IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "<pad>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_UNK_TOKEN = "<unk>"

LLAMA_EQUALS_TOKENS = set([353, 3892, 29922, 10457]) # _=, )=, =, =-
LLAMA_LEFTMARK_TOKENS = set([3532, 9314]) # <<, _<<
LLAMA_RIGHTMARK_TOKEN = 6778 # >>
LLAMA_NEWLINE_TOKEN = 13 # \n