nvidia/Cosmos3-Edge

1 2{% macro render_extra_keys(json_dict, handled_keys) %}3    {%- if json_dict is mapping %}4        {%- for json_key in json_dict if json_key not in handled_keys %}5            {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}6                {{- '7<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}8            {%- else %}9                {{-'10<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}11            {%- endif %}12        {%- endfor %}13    {%- endif %}14{% endmacro %}15{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}16{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}17 18{%- set ns = namespace(last_user_idx = -1) %}19{%- set loop_messages = messages %}20{%- for m in loop_messages %}21  {%- if m["role"] == "user" %}22    {%- set ns.last_user_idx = loop.index0 %}23  {%- endif %}24{%- endfor %}25 26{%- if messages[0]["role"] == "system" %}27    {%- if messages[0]["content"] is string %}28        {%- set system_message = messages[0]["content"] %}29    {%- else %}30        {%- set ns = namespace(system_text="") %}31        {%- for part in messages[0]["content"] %}32            {%- if 'text' in part %}33                {%- set ns.system_text = ns.system_text + part.text %}34            {%- endif %}35        {%- endfor %}36        {%- set system_message = ns.system_text %}37    {%- endif %}38 39    {%- set loop_messages = messages[1:] %}40{%- else %}41    {%- set system_message = "" %}42    {%- set loop_messages = messages %}43{%- endif %}44{%- if not tools is defined %}45    {%- set tools = [] %}46{%- endif %}47{# Recompute last_user_idx relative to loop_messages after handling system #}48{%- set ns = namespace(last_user_idx = -1) %}49{%- for m in loop_messages %}50  {%- if m["role"] == "user" %}51    {%- set ns.last_user_idx = loop.index0 %}52  {%- endif %}53{%- endfor %}54{%- if system_message is defined %}55    {{- "<|im_start|>system56" + system_message }}57{%- else %}58    {%- if tools is iterable and tools | length > 0 %}59        {{- "<|im_start|>system60" }}61    {%- endif %}62{%- endif %}63{%- if tools is iterable and tools | length > 0 %}64    {%- if system_message is defined and system_message | length > 0 %}65        {{- "66 67" }}68    {%- endif %}69    {{- "# Tools70 71You have access to the following functions:72 73" }}74    {{- "<tools>" }}75    {%- for tool in tools %}76        {%- if tool.function is defined %}77            {%- set tool = tool.function %}78        {%- endif %}79        {{- "80<function>81<name>" ~ tool.name ~ "</name>" }}82        {%- if tool.description is defined %}83            {{- '84<description>' ~ (tool.description | trim) ~ '</description>' }}85        {%- endif %}86        {{- '87<parameters>' }}88        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}89            {%- for param_name, param_fields in tool.parameters.properties|items %}90                {{- '91<parameter>' }}92                {{- '93<name>' ~ param_name ~ '</name>' }}94                {%- if param_fields.type is defined %}95                    {{- '96<type>' ~ (param_fields.type | string) ~ '</type>' }}97                {%- endif %}98                {%- if param_fields.description is defined %}99                    {{- '100<description>' ~ (param_fields.description | trim) ~ '</description>' }}101                {%- endif %}102                {%- if param_fields.enum is defined %}103                    {{- '104<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}105                {%- endif %}106                {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}107                {{- render_extra_keys(param_fields, handled_keys) }}108                {{- '109</parameter>' }}110            {%- endfor %}111        {%- endif %}112        {% set handled_keys = ['type', 'properties', 'required'] %}113        {{- render_extra_keys(tool.parameters, handled_keys) }}114        {%- if tool.parameters is defined and tool.parameters.required is defined %}115            {{- '116<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}117        {%- endif %}118        {{- '119</parameters>' }}120        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}121        {{- render_extra_keys(tool, handled_keys) }}122        {{- '123</function>' }}124    {%- endfor %}125    {{- "126</tools>" }}127 128    {{- '129 130If you choose to call a function ONLY reply in the following format with NO suffix:131 132<tool_call>133<function=example_function_name>134<parameter=example_parameter_1>135value_1136</parameter>137<parameter=example_parameter_2>138This is the value for the second parameter139that can span140multiple lines141</parameter>142</function>143</tool_call>144 145<IMPORTANT>146Reminder:147- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags148- Required parameters MUST be specified149- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after150- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls151</IMPORTANT>' }}152{%- endif %}153 154 155{%- if system_message is defined %}156    {{- '<|im_end|>157' }}158{%- else %}159    {%- if tools is iterable and tools | length > 0 %}160        {{- '<|im_end|>161' }}162    {%- endif %}163{%- endif %}164 165{%- set image_count = namespace(value=0) %}166{%- set video_count = namespace(value=0) %}167{%- for message in loop_messages %}168    {%- if message.role == "assistant" %}169        {# Add reasoning content in to content field for unified processing below. #}170        {%- set text_content = namespace(text_content="") -%}171        {%- if message.content is string %}172            {%- set text_content.text_content = message.content %}173        {%- else %}174            {%- for content in message.content %}175                {%- if 'text' in content %}176                    {%- set text_content.text_content = content.text %}177                    {%- break %}178                {%- endif %}179            {%- endfor %}180        {%- endif %}181        {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}182            {%- set content = "<think>183" ~ message.reasoning_content ~ "184</think>185" ~ (text_content.text_content | default('', true)) %}186        {%- else %}187            {%- set content = text_content.text_content | default('', true) %}188            {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}189            {%- if '<think>' not in content and '</think>' not in content %}190                {%- set content = "<think></think>" ~ content %}191            {%- else %}192                {%- set content = content %}193            {%- endif -%}194        {%- endif %}195        {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}196            {# Assistant message has tool calls. #}197            {{- '<|im_start|>assistant198' }}199                {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}200                {%- if content is string and content | trim | length > 0 %}201                    {%- if include_content %}202                        {{- (content | trim) ~ '203' -}}204                    {%- else %}205                        {%- set c = (content | string) %}206                        {%- if '</think>' in c %}207                            {# Keep only content after the last closing think. Also generation prompt causes this. #}208                            {%- set c = c.split('</think>')[-1] %}209                        {%- elif '<think>' in c %}210                            {# If <think> was opened but never closed, drop the trailing think segment #}211                            {%- set c = c.split('<think>')[0] %}212                        {%- endif %}213                        {%- set c = "<think></think>" ~ c | trim %}214                        {%- if c | length > 0 %}215                            {{- c ~ '216' -}}217                        {%- endif %}218                    {%- endif %}219                {%- else %}220                    {{- "<think></think>" -}}221                {%- endif %}222                {%- for tool_call in message.tool_calls %}223                    {%- if tool_call.function is defined %}224                        {%- set tool_call = tool_call.function %}225                    {%- endif %}226                    {{- '<tool_call>227<function=' ~ tool_call.name ~ '>228' -}}229                        {%- if tool_call.arguments is defined %}230                            {%- for args_name, args_value in tool_call.arguments|items %}231                                {{- '<parameter=' ~ args_name ~ '>232' -}}233                                    {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}234                                {{- args_value ~ '235</parameter>236' -}}237                            {%- endfor %}238                        {%- endif %}239                    {{- '</function>240</tool_call>241' -}}242                {%- endfor %}243                {{- '<|im_end|>244' }}245        {%- else %}246            {# Assistant message doesn't have tool calls. #}247            {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}248                {{- '<|im_start|>assistant249' ~ (content | default('', true) | string | trim) ~ '<|im_end|>250' }}251            {%- else %}252                {%- set c = (content | default('', true) | string) %}253                {%- if '<think>' in c and '</think>' in c %}254                    {%- set c = "<think></think>" ~ c.split('</think>')[-1] %}255                {%- endif %}256                {%- set c = c | trim %}257                {%- if c | length > 0 %}258                    {{- '<|im_start|>assistant259' ~ c ~ '<|im_end|>260' }}261                {%- else %}262                    {{- '<|im_start|>assistant263<|im_end|>264' }}265                {%- endif %}266            {%- endif %}267        {%- endif %}268    {%- elif message.role == "user" or message.role == "system" %}269        {{- '<|im_start|>' + message.role + '270' }}271        {%- if message.content is string %}272            {{- message.content }}273        {%- else %}274            {%- for content in message.content %}275                {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}276                    {%- set image_count.value = image_count.value + 1 %}277                    {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}278                    <|vision_start|><|image_pad|><|vision_end|>279                {%- elif content.type == 'video' or 'video' in content %}280                    {%- set video_count.value = video_count.value + 1 %}281                    {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}282                    <|vision_start|><|video_pad|><|vision_end|>283                {%- elif 'text' in content %}284                    {{- content.text }}285                {%- endif %}286            {%- endfor %}287        {%- endif %}288        {{- '<|im_end|>289' }}290    {%- elif message.role == "tool" %}291        {%- if loop.previtem and loop.previtem.role != "tool" %}292            {{- '<|im_start|>user293' }}294        {%- endif %}295        {{- '<tool_response>296' }}297        {{- message.content }}298        {{- '299</tool_response>300' }}301        {%- if not loop.last and loop.nextitem.role != "tool" %}302            {{- '<|im_end|>303' }}304        {%- elif loop.last %}305            {{- '<|im_end|>306' }}307        {%- endif %}308    {%- else %}309        {{- '<|im_start|>' + message.role + '310' + message.content + '<|im_end|>311' }}312    {%- endif %}313{%- endfor %}314 315{%- if add_generation_prompt %}316    {%- if enable_thinking %}317        {{- '<|im_start|>assistant318<think>319' }}320    {%- else %}321        {{- '<|im_start|>assistant322<think></think>' }}323    {%- endif %}324{%- endif %}