Felipe97/llama-cpp-compiled
01.1k
1#include "value.h"2#include "runtime.h"3#include "caps.h"4 5// note: the json dependency is only for defining input in a convenient way6// we can remove it in the future when we figure out a better way to define inputs using jinja::value7#include "json.h"8 9#include <functional>10#include <sstream>11 12#define FILENAME "jinja-caps"13 14using json = common_json;15 16namespace jinja {17 18using caps_json_fn = std::function<json()>;19using caps_ctx_fn = std::function<void(context &)>;20using caps_analyze_fn = std::function<void(context &, bool, value &, value &, const std::string &)>;21 22void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) {23 ctx.set_val("preserve_thinking", mk_val<value_bool>(enabled));24 ctx.set_val("clear_thinking", mk_val<value_bool>(!enabled));25 ctx.set_val("truncate_history_thinking", mk_val<value_bool>(!enabled));26 ctx.set_val("drop_thinking", mk_val<value_bool>(!enabled));27}28 29void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort) {30 value var = mk_val<value_string>(effort); // bind to the same value for stats31 ctx.set_val("reasoning_effort", var);32 ctx.set_val("reasoning_strength", var);33}34 35static void caps_try_execute(jinja::program & prog,36 const caps_json_fn & messages_fn,37 const caps_ctx_fn & ctx_fn,38 const caps_json_fn & tools_fn,39 const caps_analyze_fn & analyze_fn) {40 context ctx;41 ctx.is_get_stats = true;42 jinja::global_from_json(ctx, json{43 {"messages", messages_fn()},44 {"tools", tools_fn ? tools_fn() : json::array()},45 {"bos_token", ""},46 {"eos_token", ""},47 {"add_generation_prompt", true}48 }, true);49 50 if (ctx_fn) {51 ctx_fn(ctx);52 }53 54 auto messages = ctx.get_val("messages");55 auto tools = ctx.get_val("tools");56 57 bool success = false;58 std::string result;59 try {60 jinja::runtime runtime(ctx);61 auto results = runtime.execute(prog);62 auto parts = jinja::runtime::gather_string_parts(results);63 result = parts->as_string().str();64 success = true;65 } catch (const std::exception & e) {66 JJ_DEBUG("Exception during execution: %s", e.what());67 result = "";68 // ignore exceptions during capability analysis69 }70 71 analyze_fn(ctx, success, messages, tools, result);72}73 74// for debugging only75static void caps_print_stats(value & v, const std::string & path) {76 std::string ops;77 for (const auto & name : v->stats.ops) {78 ops += name + " ";79 }80 JJ_DEBUG("Value %s, type: %s %s, ops: %s",81 path.c_str(),82 v->type().c_str(),83 v->stats.used ? "(used)" : "",84 ops.c_str());85}86 87std::map<std::string, bool> caps::to_map() const {88 return {89 {"supports_string_content", supports_string_content},90 {"supports_typed_content", supports_typed_content},91 {"supports_tools", supports_tools},92 {"supports_tool_calls", supports_tool_calls},93 {"supports_parallel_tool_calls", supports_parallel_tool_calls},94 {"supports_system_role", supports_system_role},95 {"supports_preserve_reasoning", supports_preserve_reasoning},96 {"supports_reasoning_effort", supports_reasoning_effort},97 {"supports_object_arguments", supports_object_arguments},98 };99}100 101std::string caps::to_string() const {102 std::ostringstream ss;103 ss << "Caps(\n";104 for (const auto & [key, value] : to_map()) {105 ss << " " << key << "=" << (value ? "true" : "false") << "\n";106 }107 ss << ")";108 return ss.str();109}110 111caps caps_get(jinja::program & prog) {112 caps result;113 114 static const auto has_op = [](value & v, const std::string & op_name) {115 return v->stats.ops.find(op_name) != v->stats.ops.end();116 };117 118 JJ_DEBUG("%s\n", ">>> Running capability check: typed content");119 120 bool checks_for_string = false;121 static const std::string content_marker = "STRING_MARKER";122 123 // case: typed content support124 caps_try_execute(125 prog,126 [&]() {127 // messages128 return json::array({129 {130 {"role", "user"},131 {"content", content_marker}132 }133 });134 },135 nullptr, // ctx_fn136 nullptr, // tools_fn137 [&](context &, bool success, value & messages, value &, const std::string & rendered) {138 auto & content = messages->at(0)->at("content");139 caps_print_stats(content, "messages[0].content");140 if (has_op(content, "test_is_string")) {141 // checked if content is string142 checks_for_string = true;143 }144 bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");145 if (used_as_array) {146 // accessed as an array147 result.supports_typed_content = true;148 }149 if (!success) {150 // failed to execute with content as string151 result.supports_string_content = false;152 } else if (used_as_array && rendered.find(content_marker) == std::string::npos) {153 // edge case: string may be accessed for checking, but does not appear in the output154 result.supports_string_content = false;155 }156 }157 );158 159 if (checks_for_string) {160 caps_try_execute(161 prog,162 [&]() {163 // messages164 return json::array({165 {166 {"role", "user"},167 {"content", json::array({168 })}169 }170 });171 },172 nullptr, // ctx_fn173 nullptr, // tools_fn174 [&](context &, bool success, value & messages, value &, const std::string &) {175 auto & content = messages->at(0)->at("content");176 caps_print_stats(content, "messages[0].content");177 bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");178 if (used_as_array && success) {179 // accessed as an array180 result.supports_typed_content = true;181 }182 }183 );184 }185 186 JJ_DEBUG("%s\n", ">>> Running capability check: system prompt");187 188 // case: system prompt support189 caps_try_execute(190 prog,191 [&]() {192 // messages193 return json::array({194 {195 {"role", "system"},196 {"content", "System message"}197 },198 {199 {"role", "user"},200 {"content", "User message"}201 },202 });203 },204 nullptr, // ctx_fn205 nullptr, // tools_fn206 [&](context &, bool, value & messages, value &, const std::string &) {207 auto & content = messages->at(0)->at("content");208 caps_print_stats(content, "messages[0].content");209 if (!content->stats.used) {210 result.supports_system_role = false;211 }212 }213 );214 215 JJ_DEBUG("%s\n", ">>> Running capability check: single tool with object arguments support");216 217 // case: tools support: single call with object arguments218 caps_try_execute(219 prog,220 [&]() {221 // messages222 return json::array({223 {224 {"role", "user"},225 {"content", "User message"},226 },227 {228 {"role", "assistant"},229 {"content", ""}, // Some templates expect content to be empty with tool calls230 {"tool_calls", json::array({231 {232 {"id", "call00001"},233 {"type", "function"},234 {"function", {235 {"name", "tool1"},236 {"arguments", {237 {"arg", "value"}238 }}239 }}240 }241 })}242 },243 {244 {"role", "tool"},245 {"content", "Tool response"},246 {"tool_call_id", "call00001"}247 },248 {249 {"role", "assistant"},250 {"content", "The tool response was 'tool response'"}251 },252 {253 {"role", "user"},254 {"content", "User message"},255 },256 });257 },258 nullptr, // ctx_fn259 [&]() {260 // tools261 return json::array({262 {263 {"name", "tool"},264 {"type", "function"},265 {"function", {266 {"name", "tool1"},267 {"description", "Tool description"},268 {"parameters", {269 {"type", "object"},270 {"properties", {271 {"arg", {272 {"type", "string"},273 {"description", "Arg description"},274 }},275 }},276 {"required", json::array({ "arg" })},277 }},278 }},279 },280 });281 },282 [&](context &, bool success, value & messages, value & tools, const std::string &) {283 if (!success) {284 return; // Nothing can be inferred285 }286 287 auto & tool_name = tools->at(0)->at("function")->at("name");288 caps_print_stats(tool_name, "tools[0].function.name");289 caps_print_stats(tools, "tools");290 if (!tool_name->stats.used) {291 result.supports_tools = false;292 }293 294 auto & tool_calls = messages->at(1)->at("tool_calls");;295 caps_print_stats(tool_calls, "messages[1].tool_calls");296 if (!tool_calls->stats.used) {297 result.supports_tool_calls = false;298 return;299 }300 301 auto & tool_arg = tool_calls->at(0)->at("function")->at("arguments")->at("arg");302 caps_print_stats(tool_arg, "messages[1].tool_calls[0].function.arguments.arg");303 if (tool_arg->stats.used) {304 result.supports_object_arguments = true;305 }306 }307 );308 309 if (!result.supports_object_arguments) {310 JJ_DEBUG("%s\n", ">>> Running capability check: single tool with string arguments support");311 312 // case: tools support: single call with string arguments313 caps_try_execute(314 prog,315 [&]() {316 // messages317 return json::array({318 {319 {"role", "user"},320 {"content", "User message"},321 },322 {323 {"role", "assistant"},324 {"content", ""}, // Some templates expect content to be empty with tool calls325 {"tool_calls", json::array({326 {327 {"id", "call00001"},328 {"type", "function"},329 {"function", {330 {"name", "tool1"},331 {"arguments", R"({"arg": "value"})"}332 }}333 }334 })}335 },336 {337 {"role", "tool"},338 {"content", "Tool response"},339 {"tool_call_id", "call00001"}340 },341 {342 {"role", "assistant"},343 {"content", "The tool response was 'tool response'"}344 },345 {346 {"role", "user"},347 {"content", "User message"},348 },349 });350 },351 nullptr, // ctx_fn352 [&]() {353 // tools354 return json::array({355 {356 {"name", "tool"},357 {"type", "function"},358 {"function", {359 {"name", "tool1"},360 {"description", "Tool description"},361 {"parameters", {362 {"type", "object"},363 {"properties", {364 {"arg", {365 {"type", "string"},366 {"description", "Arg description"},367 }},368 }},369 {"required", json::array({ "arg" })},370 }},371 }},372 },373 });374 },375 [&](context &, bool success, value & messages, value & tools, const std::string &) {376 if (!success) {377 result.supports_tool_calls = false;378 result.supports_tools = false;379 return;380 }381 382 auto & tool_name = tools->at(0)->at("function")->at("name");383 caps_print_stats(tool_name, "tools[0].function.name");384 caps_print_stats(tools, "tools");385 if (!tool_name->stats.used) {386 result.supports_tools = false;387 }388 389 auto & tool_calls = messages->at(1)->at("tool_calls");390 caps_print_stats(tool_calls, "messages[1].tool_calls");391 if (!tool_calls->stats.used) {392 result.supports_tool_calls = false;393 return;394 }395 }396 );397 }398 399 JJ_DEBUG("%s\n", ">>> Running capability check: parallel tool support");400 401 // case: tools support: parallel calls402 caps_try_execute(403 prog,404 [&]() {405 json args = json(R"({"arg": "value"})");406 if (result.supports_object_arguments) {407 args = json{{"arg", "value"}};408 }409 410 // messages411 return json::array({412 {413 {"role", "user"},414 {"content", "User message"},415 },416 {417 {"role", "assistant"},418 {"content", ""}, // Some templates expect content to be empty with tool calls419 {"tool_calls", json::array({420 {421 {"id", "call00001"},422 {"type", "function"},423 {"function", {424 {"name", "tool1"},425 {"arguments", args}426 }}427 },428 {429 {"id", "call00002"},430 {"type", "function"},431 {"function", {432 {"name", "tool1"},433 {"arguments", args}434 }}435 }436 })}437 },438 {439 {"role", "tool"},440 {"content", "Tool response"},441 {"tool_call_id", "call00001"}442 },443 {444 {"role", "assistant"},445 {"content", "The tool response was 'tool response'"}446 },447 {448 {"role", "user"},449 {"content", "User message"},450 },451 });452 },453 nullptr, // ctx_fn454 [&]() {455 // tools456 return json::array({457 {458 {"name", "tool"},459 {"type", "function"},460 {"function", {461 {"name", "tool1"},462 {"description", "Tool description"},463 {"parameters", {464 {"type", "object"},465 {"properties", {466 {"arg", {467 {"type", "string"},468 {"description", "Arg description"},469 }},470 }},471 {"required", json::array({ "arg" })},472 }},473 }},474 },475 });476 },477 [&](context &, bool success, value & messages, value &, const std::string &) {478 if (!success) {479 result.supports_parallel_tool_calls = false;480 return;481 }482 483 auto & tool_calls = messages->at(1)->at("tool_calls");484 caps_print_stats(tool_calls, "messages[1].tool_calls");485 486 // check for second tool call usage487 auto & tool_call_1 = tool_calls->at(1)->at("function");488 caps_print_stats(tool_call_1, "messages[1].tool_calls[1].function");489 if (!tool_call_1->stats.used) {490 result.supports_parallel_tool_calls = false;491 }492 }493 );494 495 JJ_DEBUG("%s\n", ">>> Running capability check: preserve reasoning");496 497 // case: preserve reasoning content in chat history498 const std::string reasoning_placeholder = "<REASONING_CONTENT_PLACEHOLDER>";499 caps_try_execute(500 prog,501 [&]() {502 // messages503 return json::array({504 {505 {"role", "user"},506 {"content", "User message"}507 },508 {509 {"role", "assistant"},510 {"content", "Assistant message"},511 // check of reasoning_content deeper in the history, not just the last assistant message512 {"reasoning_content", reasoning_placeholder}513 },514 {515 {"role", "user"},516 {"content", "User message"}517 },518 {519 {"role", "assistant"},520 {"content", "Assistant message"},521 {"reasoning_content", "Reasoning content"}522 },523 {524 {"role", "user"},525 {"content", "User message"}526 },527 });528 },529 [&](context & ctx) {530 ctx.set_val("enable_thinking", mk_val<value_bool>(true));531 caps_apply_preserve_reasoning(ctx, true);532 },533 nullptr, // tools_fn534 [&](context &, bool, value &, value &, const std::string & output) {535 // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result536 if (output.find(reasoning_placeholder) != std::string::npos) {537 result.supports_preserve_reasoning = true;538 }539 }540 );541 542 JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort");543 544 // case: reasoning effort level545 caps_try_execute(546 prog,547 [&]() {548 // messages549 return json::array({550 {551 {"role", "user"},552 {"content", "User message"}553 },554 });555 },556 [&](context & ctx) {557 ctx.set_val("enable_thinking", mk_val<value_bool>(true));558 caps_apply_reasoning_effort(ctx, "low");559 },560 nullptr, // tools_fn561 [&](context & ctx, bool, value &, value &, const std::string &) {562 value effort = ctx.get_val("reasoning_effort");563 caps_print_stats(effort, "reasoning_effort");564 result.supports_reasoning_effort = effort->stats.used;565 }566 );567 568 JJ_DEBUG("%s\n", result.to_string().c_str());569 570 return result;571}572 573} // namespace jinja574