andrewhinh commited on
Commit
39423d0
·
verified ·
1 Parent(s): 5b363a4

Upload processor

Browse files
Files changed (2) hide show
  1. chat_template.json +3 -0
  2. tokenizer_config.json +1 -1
chat_template.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
3
+ }
tokenizer_config.json CHANGED
@@ -134,7 +134,7 @@
134
  "clean_up_tokenization_spaces": false,
135
  "eos_token": "<|im_end|>",
136
  "errors": "replace",
137
- "model_max_length": 32768,
138
  "pad_token": "<|endoftext|>",
139
  "padding_side": "right",
140
  "processor_class": "Qwen2VLProcessor",
 
134
  "clean_up_tokenization_spaces": false,
135
  "eos_token": "<|im_end|>",
136
  "errors": "replace",
137
+ "model_max_length": 1024,
138
  "pad_token": "<|endoftext|>",
139
  "padding_side": "right",
140
  "processor_class": "Qwen2VLProcessor",