/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /** * @file test_frame_capture.c * @brief Tests for FRAME CAPTURE handling in the tool-call parser * (src/serve/tool_calling.c). * * When a turn exhausts its `reasoning_budget` the serve injects the * "" token itself; the grammar cannot tell that close from the * model's own, so FORCED mode compels the tool-call opener while the model * is still mid-thought. Whatever it was still going to say — often a * complete tool call — lands in the free-text parameter VALUE. * * Every FX_* literal comes from tests/frame_capture_fixtures.h: synthetic * content in the shapes observed in real serving. FX_NATIVE_XML_CAPTURE * is the control that rules the dialect explanation out: the captured * text there is a NATIVE Qwen-XML call. * * The suite asserts three behaviours: * RECOVER — a value that IS a complete call becomes that call, with the * exact tool and arguments the model asked for. * REJECT — a value beginning with the reasoning close (and only that) * surfaces no call at all. * PASS — everything else, including the captured scratch-text values * from the same sessions, is untouched. * * Pure C string parsing, no model/engine deps — fully local. * * @status None * @issues None * @todo None */ #include "lib/test.h" #include "frame_capture_fixtures.h" #include "../src/serve/tool_calling.h" #include "../src/serve/json_reader.h" #include #include #include /* ============================================================ * Tool schemas — a typical agent client's shapes * ============================================================ */ static const char *READ_TOOLS = "[{\"type\":\"function\",\"function\":{\"name\":\"read\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}," "\"required\":[\"path\"]}}}]"; static const char *GLOB_TOOLS = "[{\"type\":\"function\",\"function\":{\"name\":\"glob\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"pattern\":{\"type\":\"string\"}}," "\"required\":[\"pattern\"]}}}]"; /* grep + read: the FX_NATIVE_XML_CAPTURE shape is a `read` call captured inside a * `grep` frame, so both must be declared for that recovery to be legal. */ static const char *GREP_READ_TOOLS = "[{\"type\":\"function\",\"function\":{\"name\":\"grep\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"pattern\":{\"type\":\"string\"}}," "\"required\":[\"pattern\"]}}}," "{\"type\":\"function\",\"function\":{\"name\":\"read\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}," "\"required\":[\"path\"]}}}]"; /* read + write, so a two-parameter call with a big free-text `content` can be * exercised: that is the argument most likely to legitimately MENTION the * frame's vocabulary (an agent editing this very source file). */ static const char *WRITE_TOOLS = "[{\"type\":\"function\",\"function\":{\"name\":\"read\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}," "\"required\":[\"path\"]}}}," "{\"type\":\"function\",\"function\":{\"name\":\"write\",\"parameters\":" "{\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}," "\"content\":{\"type\":\"string\"}}," "\"required\":[\"path\",\"content\"]}}}]"; /* ============================================================ * Helpers * ============================================================ */ /* Rebuild the raw model emission that produces an argument: the compelled * wrapped XML frame with `value` sitting in the parameter's VALUE slot. The * fixture is the post-dup_trim value, so re-wrapping it reproduces the bytes * the parser sees. */ static char *xml_frame(const char *tool, const char *key, const char *value) { size_t n = strlen(tool) + strlen(key) + strlen(value) + 128; char *buf = malloc(n); if (!buf) return NULL; snprintf(buf, n, "\n\n\n%s\n\n" "\n", tool, key, value); return buf; } /* Parse one reconstructed frame under the QWEN_XML dialect. */ static int parse_frame(const char *tools_json, const char *tool, const char *key, const char *value, tc_tool_call **calls, int *n) { jr_value *tools = jr_parse(tools_json); char *text = xml_frame(tool, key, value); int rc = tc_parse_tool_calls(text, tools, TC_DIALECT_QWEN_XML, calls, n); free(text); jr_free(tools); return rc; } /* ============================================================ * RECOVER — captured calls * ============================================================ */ /* THE CONTROL. A native Qwen-XML `read` call captured inside a `grep` frame. * Recovering this needs no theory about dialects at all — the captured call is * in the grammar's own dialect. */ static void test_native_xml_capture_recovered(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(GREP_READ_TOOLS, "grep", "pattern", FX_NATIVE_XML_CAPTURE, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_NOT_NULL(calls); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"lumen/serve.log\"}"); ASSERT_TRUE(strncmp(calls[0].id, "call_", 5) == 0); tc_tool_calls_free(calls, n); } static void test_hermes_leadin_capture_recovered(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", FX_HERMES_LEADIN_CAPTURE, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_NOT_NULL(calls); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"lumen/Makefile\"}"); ASSERT_TRUE(strncmp(calls[0].id, "call_", 5) == 0); tc_tool_calls_free(calls, n); } static void test_overclosed_capture_recovered(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", FX_OVERCLOSED_CAPTURE, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"lumen/include/lumen/settings_cache.h\"}"); tc_tool_calls_free(calls, n); } static void test_absolute_path_capture_recovered(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", FX_ABSOLUTE_PATH_CAPTURE, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"/home/user/src/lumen/include/settings_cache.h\"}"); tc_tool_calls_free(calls, n); } /* The recovered call must carry no trace of the outer frame. */ static void test_recovered_arguments_drop_the_outer_frame(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", FX_HERMES_LEADIN_CAPTURE, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_NULL(strstr(calls[0].arguments, "")); ASSERT_NULL(strstr(calls[0].arguments, "")); ASSERT_NULL(strstr(calls[0].arguments, "call_function")); tc_tool_calls_free(calls, n); } /* A capture that names a DIFFERENT tool than the armed frame routes to the tool * the model actually asked for — that is the whole point. */ static void test_capture_across_tools_routes_to_the_named_tool(void) { const char *v = "call_function{\"func\":\"write\",\"args\":" "{\"path\":\"/tmp/a.txt\",\"content\":\"hi\"}}"; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(WRITE_TOOLS, "read", "path", v, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "write"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"/tmp/a.txt\",\"content\":\"hi\"}"); tc_tool_calls_free(calls, n); } /* ============================================================ * REJECT — a leading reasoning close, and nothing else * ============================================================ */ /* The value is the force-closed model's own "" plus an unfinished * thought: unmistakably a captured frame, with no call in it. No tool call is * surfaced rather than dispatching the fragment as a path. */ static void test_leading_reasoning_close_rejected(void) { const char *v = "\n\nI should look at the Makefile next."; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", v, &calls, &n), 0); ASSERT_INT_EQ(n, 0); tc_tool_calls_free(calls, n); } /* An inner call naming a tool the request never declared is NOT recovered — the * tool_name_known gate holds and no invented call reaches the client. */ static void test_undeclared_inner_tool_never_recovered(void) { const char *v = "call_function{\"func\":\"rm_rf\",\"args\":{\"path\":\"/\"}}"; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", v, &calls, &n), 0); /* Not recovered, and NOT rejected either: "" is not a rejecting * marker (that collision with real `write`/`edit` content is untested), so * the outer call passes through as the harmless read it already was. */ ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_NULL(strstr(calls[0].arguments, "rm_rf\":")); tc_tool_calls_free(calls, n); } /* ============================================================ * PASS — captured scratch text, and legitimate content * ============================================================ */ /* A bash pipeline captured into a `read` path. The model * wanted `bash`, but the value carries no frame marker and no complete call, so * there is nothing to recover — the call passes through exactly as before. * Recovering it would mean GUESSING the tool from shell metacharacters, which * misfires on legitimate `bash`/`grep` arguments. */ static void test_shell_in_read_path_untouched(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", FX_SHELL_IN_READ_PATH, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "settings_cache.c.md 2>/dev/null")); tc_tool_calls_free(calls, n); } /* Shell pipeline + dangling fence in a `glob` pattern. */ static void test_shell_in_glob_pattern_untouched(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(GLOB_TOOLS, "glob", "pattern", FX_SHELL_IN_GLOB, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "glob"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "lume-4471093821")); tc_tool_calls_free(calls, n); } /* Over 5,000 chars of analysis, C source with braces, and a * truncated SEARCH/REPLACE block. The braces must NOT be mistaken for a JSON * call — exactly one `glob` call comes out, unchanged. */ static void test_analysis_dump_untouched(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_TRUE(strlen(FX_ANALYSIS_DUMP) > 5000); ASSERT_INT_EQ(parse_frame(GLOB_TOOLS, "glob", "pattern", FX_ANALYSIS_DUMP, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "glob"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "<<<<<<< SEARCH")); tc_tool_calls_free(calls, n); } /* THE common-case guard: an agent writing a file whose CONTENT mentions the * frame's vocabulary (this repo's own sources do exactly that). No complete * call runs to the end, so the write is dispatched intact. */ static void test_write_content_mentioning_the_frame_is_kept(void) { const char *v = "/* the parser watches these */\n" "static const char TC_OPEN[] = \"\";\n" "static const char FN_OPEN[] = \"" rejects. */ static void test_write_content_starting_with_frame_literal_is_kept(void) { const char *v = " is the wrapper opener this parser watches for."; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(WRITE_TOOLS, "write", "content", v, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "write"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "wrapper opener")); tc_tool_calls_free(calls, n); } /* The "runs to the end" gate: a value that CONTAINS a complete call but keeps * going afterwards is documentation, not a capture. */ static void test_value_containing_a_call_as_content_is_kept(void) { const char *v = "for example call_function{\"func\":\"read\",\"args\":" "{\"path\":\"/x\"}} is what that shape looks like in practice"; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(WRITE_TOOLS, "write", "content", v, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "write"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "in practice")); tc_tool_calls_free(calls, n); } /* A plain JSON object in a value, with NO frame marker, is never inspected — * a `write` payload that happens to be JSON stays a write. */ static void test_plain_json_value_without_marker_is_kept(void) { const char *v = "{\"func\": \"read\", \"args\": {\"path\": \"/etc/hostname\"}}"; tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(WRITE_TOOLS, "write", "content", v, &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "write"); ASSERT_NOT_NULL(strstr(calls[0].arguments, "etc/hostname")); tc_tool_calls_free(calls, n); } /* Ordinary turns must still parse: an ordinary call is completely unaffected * by the capture path. */ static void test_ordinary_call_unaffected(void) { tc_tool_call *calls = NULL; int n = -1; ASSERT_INT_EQ(parse_frame(READ_TOOLS, "read", "path", "lumen/src/settings/settings_cache.c", &calls, &n), 0); ASSERT_INT_EQ(n, 1); ASSERT_STR_EQ(calls[0].name, "read"); ASSERT_STR_EQ(calls[0].arguments, "{\"path\":\"lumen/src/settings/settings_cache.c\"}"); tc_tool_calls_free(calls, n); } /* ============================================================ * Main * ============================================================ */ int main(void) { printf("\n=== Forced-close FRAME CAPTURE ===\n\n"); RUN_TEST(native_xml_capture_recovered); RUN_TEST(hermes_leadin_capture_recovered); RUN_TEST(overclosed_capture_recovered); RUN_TEST(absolute_path_capture_recovered); RUN_TEST(recovered_arguments_drop_the_outer_frame); RUN_TEST(capture_across_tools_routes_to_the_named_tool); RUN_TEST(leading_reasoning_close_rejected); RUN_TEST(undeclared_inner_tool_never_recovered); RUN_TEST(shell_in_read_path_untouched); RUN_TEST(shell_in_glob_pattern_untouched); RUN_TEST(analysis_dump_untouched); RUN_TEST(write_content_mentioning_the_frame_is_kept); RUN_TEST(write_content_starting_with_frame_literal_is_kept); RUN_TEST(value_containing_a_call_as_content_is_kept); RUN_TEST(plain_json_value_without_marker_is_kept); RUN_TEST(ordinary_call_unaffected); TEST_REPORT(); return __ft_failed > 0 ? 1 : 0; }