/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /* * Unit tests for the tool-call decoding grammar (src/serve/tool_grammar.c). * * The grammar is the Phase-5 correctness gate: with a tools schema it must * ACCEPT the model's trained XML emission (wrapped … and the bare * #include #include /* Two tools: glob(pattern, path) and read(path). No `required` list on either, * so these stay the fixture for everything that is not about requiredness. * * Deliberately NO "minLength" on any parameter: this is the schema shape that * says nothing about emptiness, and the grammar must not invent a floor it was * never given. CLIENT_FLOORED_JSON below is the same tools as the client really sends * them, and is what the empty-value tests use. */ static const char *TOOLS_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"glob\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"pattern\":{\"type\":\"string\"},\"path\":{\"type\":\"string\"}}}}}," "{\"type\":\"function\",\"function\":{\"name\":\"read\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"path\":{\"type\":\"string\"}}}}}]"; /* The client's REAL shapes for the two tools this contract turns on * (the client's schema code), verbatim in the one keyword * that matters: * * bash.command required, "minLength":1 → empty is the live failure * edit.new_string required, NO minLength → empty is the DELETION * * Both keys are `required`. That is the whole reason requiredness cannot be the * empty-value predicate and `minLength` is. */ static const char *CLIENT_FLOORED_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"bash\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"command\":{\"type\":\"string\",\"minLength\":1}}," "\"required\":[\"command\"]}}}," "{\"type\":\"function\",\"function\":{\"name\":\"edit\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"path\":{\"type\":\"string\",\"minLength\":1}," "\"old_string\":{\"type\":\"string\",\"minLength\":1}," "\"new_string\":{\"type\":\"string\"}}," "\"required\":[\"path\",\"old_string\",\"new_string\"]}}}]"; /* The client's REAL switch_mode schema (the client's schema code) — the * live victim of the dropped-`enum` gap: before the enum was compiled, a VALUE * of "banana" was a complete, well-formed switch_mode call. */ static const char *SWITCH_MODE_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"switch_mode\"," "\"parameters\":{\"type\":\"object\",\"properties\":" "{\"mode\":{\"type\":\"string\"," "\"enum\":[\"chat\",\"plan\",\"act\",\"orchestrate\"]}," "\"reason\":{\"type\":\"string\"}}," "\"required\":[\"mode\",\"reason\"]}}}]"; /* log(msg REQUIRED, level enum OPTIONAL) — an enum on a key the model may * legally leave out entirely. */ static const char *ENUM_OPTIONAL_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"log\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"msg\":{\"type\":\"string\"}," "\"level\":{\"type\":\"string\",\"enum\":[\"info\",\"warn\",\"error\"]}}," "\"required\":[\"msg\"]}}}]"; /* Enum shapes that must DEGRADE to a free-text VALUE rather than reject: * non-string members, one non-string among strings, an empty member, an empty * list, and string members contradicting a non-string `type`. `notype` has * string members and no declared type — that one DOES compile. */ static const char *ENUM_DEGRADE_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"cfg\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"nums\":{\"type\":\"integer\",\"enum\":[1,2,3]}," "\"mixed\":{\"type\":\"string\",\"enum\":[\"a\",7]}," "\"blank\":{\"type\":\"string\",\"enum\":[\"a\",\"\"]}," "\"nolist\":{\"type\":\"string\",\"enum\":[]}," "\"clash\":{\"type\":\"integer\",\"enum\":[\"a\",\"b\"]}," "\"notype\":{\"enum\":[\"yes\",\"no\"]}}}}}]"; /* Feed a whole string through the grammar one CHAR at a time, checking that * every char stays allowed (mirrors the per-token mask at char granularity — * the strictest possible feed). Returns the final state (caller frees). */ static tg_state *feed_ok(tg_grammar *g, const char *s, bool *all_allowed) { tg_state *st = tg_state_new(g); char one[2] = {0, 0}; *all_allowed = true; for (const char *p = s; *p; p++) { one[0] = *p; if (!tg_token_allowed(st, one)) { *all_allowed = false; break; } tg_advance(st, one); } return st; } /* Does feeding `s` (char by char) hit a DISALLOWED char at some point? */ static bool feed_rejects(tg_grammar *g, const char *s) { bool ok; tg_state *st = feed_ok(g, s, &ok); tg_state_free(st); return !ok; } /* Forced-mode variants. A forced state (required-any or pinned function) with * reasoning already ended — the opener is compelled from the first char. */ static tg_state *feed_forced(tg_state *st, const char *s, bool *all_allowed) { char one[2] = {0, 0}; *all_allowed = true; for (const char *p = s; *p; p++) { one[0] = *p; if (!tg_token_allowed(st, one)) { *all_allowed = false; break; } tg_advance(st, one); } return st; } /* Feed `s` into a forced (required-any) state whose reasoning has ended; does it * get rejected somewhere? */ static bool forced_rejects(tg_grammar *g, const char *s) { tg_state *st = tg_state_new_forced(g, NULL); tg_reasoning_end(st); bool ok; feed_forced(st, s, &ok); tg_state_free(st); return !ok; } /* ============================================================ * ACCEPT: canonical wrapped call. * ============================================================ */ static void test_wrapped_accepts(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); ASSERT_NOT_NULL(g); const char *call = "\n\n" "\n*.md\n\n" "\n"; bool ok; tg_state *st = feed_ok(g, call, &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * ACCEPT: bare (no wrapper) call — the Q3 model's favored form. * ============================================================ */ static void test_bare_accepts(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); const char *call = "\n\nREADME.md\n\n"; bool ok; tg_state *st = feed_ok(g, call, &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * ACCEPT: prose before the call (reasoning region) is unconstrained. * ============================================================ */ static void test_prose_then_call(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); const char *txt = "Let me look. I'll list the files < first > maybe. " "\n\nsrc\n\n"; bool ok; tg_state *st = feed_ok(g, txt, &ok); ASSERT_TRUE(ok); /* prose stayed FREE, call completed */ ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * ACCEPT: pure prose answer, NO tool call → never forced into a call. * ============================================================ */ static void test_pure_prose_no_call(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); const char *txt = "The answer is 42. No tools needed here."; bool ok; tg_state *st = feed_ok(g, txt, &ok); ASSERT_TRUE(ok); /* every char allowed */ ASSERT_FALSE(tg_in_call(st)); /* still FREE, not inside a call */ ASSERT_FALSE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * REJECT: the off-contract JSON habit {"tool_call":…}. * Once the model opens "") the skeleton engages; * feeding the JSON form as if it were a call must be rejected. We test that * the JSON, fed after an opener, breaks — i.e. the grammar won't emit it. * ============================================================ */ static void test_json_form_rejected(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* After the opener the grammar demands a NAME then '>'; JSON braces/quotes * are not part of any tool name, so this is rejected. */ ASSERT_TRUE(feed_rejects(g, "{...}" — after "\n" * the grammar demands "\n{\"tool_call\":")); tg_free(g); jr_free(tools); } /* ============================================================ * REJECT: unknown function name. * ============================================================ */ static void test_unknown_function_rejected(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* "grep" is not a tool → the 'g' is live (glob starts with g) but the name * cannot terminate as "grep", and 'r' after "g" is not live (glob → 'l'). */ ASSERT_TRUE(feed_rejects(g, "")); /* a name that is a valid prefix but closed early ("glo>") is not exact. */ ASSERT_TRUE(feed_rejects(g, "")); tg_free(g); jr_free(tools); } /* ============================================================ * REJECT: unknown parameter key for a valid function. * ============================================================ */ static void test_unknown_param_rejected(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* read has only "path"; "mode" is not a key. */ ASSERT_TRUE(feed_rejects(g, "\n")); /* glob's key "pattern" mistyped as "patern" is rejected. */ ASSERT_TRUE(feed_rejects(g, "\n")); tg_free(g); jr_free(tools); } /* ============================================================ * ACCEPT: a name/key split across "tokens" still matches (cross-boundary). * ============================================================ */ static void test_cross_token_name(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new(g); /* opener + partial name in one token, rest of the name in the next. */ ASSERT_TRUE(tg_token_allowed(st, "\n")); tg_advance(st, "ob>\n"); ASSERT_TRUE(tg_token_allowed(st, "\n")); tg_advance(st, "\n"); ASSERT_TRUE(tg_token_allowed(st, "x\n\n")); tg_advance(st, "x\n\n"); ASSERT_TRUE(tg_token_allowed(st, "")); tg_advance(st, ""); ASSERT_TRUE(tg_complete(st)); /* an unknown-name split must still be rejected at the divergent char. */ tg_state *bad = tg_state_new(g); tg_advance(bad, "")); /* "glee" diverges at 2nd 'e' */ tg_state_free(st); tg_state_free(bad); tg_free(g); jr_free(tools); } /* ============================================================ * ACCEPT: multi-parameter call (two blocks). * ============================================================ */ static void test_multi_param(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); const char *call = "\n" "\n*.c\n\n" "\nsrc\n\n" ""; bool ok; tg_state *st = feed_ok(g, call, &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * FORCED mode: after reasoning, prose/JSON are REJECTED from the start — * the opener is compelled (this is the Phase-5 gap being closed). * ============================================================ */ static void test_forced_rejects_prose_and_json(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* Plain prose refusal — the classic Q3 failure — must be masked out. */ ASSERT_TRUE(forced_rejects(g, "I cannot do that.")); /* The off-contract JSON drift, from the very first char. */ ASSERT_TRUE(forced_rejects(g, "{\"tool_call\":{\"name\":\"glob\"}}")); /* Even a stray non-'<' leading char is rejected (opener required). */ ASSERT_TRUE(forced_rejects(g, "Sure! ")); tg_free(g); jr_free(tools); } /* FORCED mode ACCEPTS a valid call fed from the start (no reasoning). */ static void test_forced_accepts_call(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); tg_reasoning_end(st); const char *call = "\n\nREADME.md\n\n"; bool ok; feed_forced(st, call, &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* FORCED mode: reasoning is allowed FIRST (before tg_reasoning_end); then the * opener is compelled. Mirrors the serve loop: content is unconstrained * until "" flips the phase, after which only the call is valid. */ static void test_forced_reasoning_then_call(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; /* Reasoning phase: arbitrary prose is allowed (mask inactive here). */ feed_forced(st, "The user wants a file listing. Let me use glob.", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_mask_active(st)); /* not yet forced */ /* "" fires reasoning-end → opener now required. */ tg_reasoning_end(st); ASSERT_TRUE(tg_mask_active(st)); /* forced: post-reasoning FREE is masked */ /* Now a valid call is accepted; prose would be rejected. */ ASSERT_TRUE(tg_token_allowed(st, "")); ASSERT_FALSE(tg_token_allowed(st, "Actually, no.")); feed_forced(st, "\n\n*.md\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); ASSERT_FALSE(tg_mask_active(st)); /* complete → constraint released */ tg_state_free(st); tg_free(g); jr_free(tools); } /* FORCED-FUNCTION mode pins the NAME: the target function is accepted; any other * tool name is rejected even though it is a valid tool. */ static void test_forced_function_pins_name(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* Pin "read": a "read" call is accepted, a "glob" call is rejected. */ tg_state *good = tg_state_new_forced(g, "read"); ASSERT_NOT_NULL(good); tg_reasoning_end(good); bool ok; feed_forced(good, "\n\nx\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(good)); tg_state_free(good); tg_state *bad = tg_state_new_forced(g, "read"); tg_reasoning_end(bad); feed_forced(bad, "", &ok); ASSERT_FALSE(ok); /* "glob" rejected: name pinned to "read" */ tg_state_free(bad); /* A pin that is not a known tool → NULL (serve falls back to AUTO). */ ASSERT_NULL(tg_state_new_forced(g, "not_a_tool")); tg_free(g); jr_free(tools); } /* ============================================================ * BUG 1 — forcing must engage from the grammar's OWN "" detection, * WITHOUT any external tg_reasoning_end() call (the emitter's flag is * unreliable on the Q3 model). The grammar watches its own token stream: * reasoning text flows freely, then "" self-fires the forced constraint, * and the model is compelled into a valid call. * ============================================================ */ static void test_forced_self_detects_think_close(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; /* Reasoning phase (mask inactive) — arbitrary prose is allowed. */ feed_forced(st, "I should read the file to answer.", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_mask_active(st)); /* grace: not yet forced */ /* The model emits "" — the grammar detects it ITSELF (no external * tg_reasoning_end call) and flips into the forced constraint. */ feed_forced(st, "", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_mask_active(st)); /* self-detected close → opener compelled */ /* Post-close: prose is now impossible, a valid call is compelled. */ ASSERT_FALSE(tg_token_allowed(st, "Sorry, I can't.")); ASSERT_TRUE(tg_token_allowed(st, "")); feed_forced(st, "\n\nREADME.md\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* BUG 1 — "" split across token boundaries is still detected (the * rolling tracker survives token splits, like a real BPE stream). */ static void test_forced_think_close_split_tokens(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; feed_forced(st, "reasoning": still grace */ feed_forced(st, "ink>", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_mask_active(st)); /* completed across the split */ feed_forced(st, "\n\nx\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* BUG 1 — a call opened DIRECTLY during reasoning (no "" first) still * engages the skeleton (auto-like), so the model's own well-formed call is * honored even when it never closes think. */ static void test_forced_direct_call_no_think(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; /* No external reasoning-end; the model opens a call straight away. */ feed_forced(st, "\n\nREADME.md\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); /* skeleton engaged mid-grace, call completed */ tg_state_free(st); tg_free(g); jr_free(tools); } /* AUTO mode is UNCHANGED by the forced-mode work: prose stays FREE, and the mask * only arms inside a call (tg_mask_active == tg_in_call). */ static void test_auto_mode_unchanged(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new(g); tg_reasoning_end(st); /* no-op in AUTO mode */ bool ok; feed_forced(st, "The answer is 42, no tools needed.", &ok); ASSERT_TRUE(ok); /* prose unconstrained */ ASSERT_FALSE(tg_mask_active(st)); /* AUTO: not inside a call → no mask */ ASSERT_FALSE(tg_in_call(st)); /* Once a call opens, the mask arms (unchanged auto behavior). */ tg_advance(st, ""); ASSERT_TRUE(tg_in_call(st)); ASSERT_TRUE(tg_mask_active(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * EMPTY VALUE: the parameter CLOSES, and the call's fate is decided by whether * the model went on to close the CALL — not by how much whitespace the value * held. * * THIS TEST USED TO ASSERT THE OPPOSITE (test_empty_value_rejected): that an * empty/whitespace value could never complete a call, because TG_VALUE refused to * let the terminator start until a non-whitespace char had been seen. That * guarantee was never real, and holding it here hid the defect it caused: * * • Refusing to START the terminator is done by CONSUMING those chars as * content, which is an ACCEPT. tg_token_allowed("\n\n") returned * true at an empty value, so nothing was ever masked and the model emitted it * freely — 9 times out of 9 on the live fleet. * • Once eaten, the "" broke the terminator match and was swallowed * as value text, so the walk stayed in TG_VALUE for the rest of the turn. * The parser's strstr found that SAME "" and closed the value at * it. Grammar and parser disagreed about a byte sequence both had seen, and * the argument vanished: `read {}`, `write` without its `content`. * * An empty value is also legal on its own terms — `edit`'s `new_string:""` is a * deletion (the client's schema code argument_is_absent) — so rejecting it * would break a correct call to catch an incorrect one that the reject does not * actually catch. The truncation signal moved to where it is decidable: * tg_complete(). * * SCOPE, since a floor now exists. Everything below runs on TOOLS_JSON, whose * parameters declare no `minLength` — the schema shape that says nothing about * emptiness. A schema that DOES say so is the next test, and the two together * are the contract: the grammar refuses an empty value where the schema forbids * one, and nowhere else. * ============================================================ */ static void test_empty_value_closes_the_parameter(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); bool ok; /* An empty value, with the model going on to close the call: accepted AND * complete. The parameter ended at "", exactly where the parser * ends it, and the call ended at "". */ tg_state *empty = feed_ok(g, "\n\n\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(empty)); tg_state_free(empty); /* THE DEFECT SHAPE — the same empty value, but the model STOPPED after the * close instead of ending the call. Every char is still on-grammar (nothing * to mask: it is a legal prefix), and the walk is back on the SKELETON rather * than stranded inside the value, so the incompleteness is visible. */ tg_state *cut = feed_ok(g, "\n\n\n\n", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_complete(cut)); /* the CALL never closed → truncation */ ASSERT_TRUE(tg_in_call(cut)); /* and it is inside a call, not adrift */ tg_state_free(cut); /* A single content char before the terminator IS accepted (non-empty). */ tg_state *st = feed_ok(g, "\n\nx\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); /* Whitespace-only values in their several shapes — all close the parameter, * none strand the walk. The old guard treated these as content and lost the * close; the value the parser reads out of each is "" either way. */ tg_state *st2 = feed_ok(g, "\n\n\n\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st2)); tg_state_free(st2); tg_state *st3 = feed_ok(g, "\n\n \n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st3)); tg_state_free(st3); tg_state *st4 = feed_ok(g, "\n\n \t \n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st4)); tg_state_free(st4); /* CONTROL: leading/trailing whitespace around REAL content is still fine — * the fix must not reject values that merely have padding, or every natural * "\n\n" emission would break. */ tg_state *st5 = feed_ok(g, "\n\n /tmp/x.c \n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st5)); tg_state_free(st5); /* CONTROL: a value that CONTAINS the terminator's leading newline but keeps * going still runs to the real close — the terminator tracker must re-seed, * not commit, on a partial match. */ tg_state *st6 = feed_ok(g, "\n\na\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st6)); tg_state_free(st6); tg_free(g); jr_free(tools); } /* ============================================================ * FLOORED VALUE: "" is OFF-GRAMMAR while the value is empty. * * THE LIVE FAILURE THIS EXISTS FOR. qwen36-bf16, tool grammar armed and masking * (masked_steps non-zero), emitted * * [tool_call bash] {"command":""} * [tool_result bash] (no output) * * SEVEN consecutive times — seven full decodes at ~10 tok/s producing nothing — * until the run hit its no-progress guard and stopped. A model that emits an * empty required argument does not recover on its own: the empty call teaches it * nothing, so it emits the same call again. That is why this is prevented at * emission and not corrected afterwards. * * WHY THE ASSERTION IS `feed_rejects` AND NOT `!tg_complete`. Those are two * different guarantees and only one of them masks a token. The guard removed in * 08f78d6 satisfied `!tg_complete` — it kept the walk from ever finishing — while * ACCEPTING every char, so the model emitted the empty value anyway and the walk * was stranded mid-value for the rest of the turn. Requiring the char to be * REFUSED is requiring that tg_token_allowed says no, which is the only form of * this constraint that changes what the model can draw. * ============================================================ */ static void test_empty_value_off_grammar_when_floored(void) { jr_value *tools = jr_parse(CLIENT_FLOORED_JSON); tg_grammar *g = tg_build(tools); bool ok; /* The live shape: bash.command, closed around nothing. */ ASSERT_TRUE(feed_rejects(g, "\n\n")); /* The unpadded spelling, and the doubled-newline one. */ ASSERT_TRUE(feed_rejects(g, "\n")); ASSERT_TRUE(feed_rejects(g, "\n\n\n")); /* Whitespace is not content: the parser trims it, so a floor satisfied by * bytes the host deletes is not a floor. */ ASSERT_TRUE(feed_rejects(g, "\n\n \n")); ASSERT_TRUE(feed_rejects(g, "\n\n \t \r\n")); /* CONTROL: one real char is enough, and the call completes. */ tg_state *good = feed_ok(g, "\n\nls\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(good)); tg_state_free(good); /* CONTROL: padding around real content is untouched. */ tg_state *padded = feed_ok(g, "\n\n ls -la \n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(padded)); tg_state_free(padded); /* CONTROL: content that merely BEGINS like the terminator is real content, * so the close after it is legal. The floor credits a broken terminator * prefix retroactively; without that these would be read as empty. */ tg_state *lt = feed_ok(g, "\n\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(lt)); tg_state_free(lt); tg_state *lone = feed_ok(g, "\n\n<\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(lone)); tg_state_free(lone); tg_free(g); jr_free(tools); } /* ============================================================ * THE OTHER SIDE OF THE SAME PREDICATE: a required string with NO minLength may * still be empty, and `edit`'s new_string is exactly that key. * * Deleting a line is an ordinary operation and `new_string:""` is how it is * spelled. A guard keyed on `required` — or an unconditional one, which is what * the JSON dialect shipped for a while — makes that call unemittable, and under * tool_choice=required an unemittable correct call is a wedge in its own right. * The floored siblings in the SAME tool are still refused, so the verdict is per * key and not the guard being off. * ============================================================ */ static void test_unfloored_value_may_still_be_empty(void) { jr_value *tools = jr_parse(CLIENT_FLOORED_JSON); tg_grammar *g = tg_build(tools); bool ok; tg_state *del = feed_ok(g, "\n\n/tmp/x.c\n\n" "\ndrop me\n\n" "\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(del)); tg_state_free(del); ASSERT_TRUE(feed_rejects(g, "\n\n")); ASSERT_TRUE(feed_rejects(g, "\n\n/tmp/x.c\n\n" "\n")); tg_free(g); jr_free(tools); } /* The close is honoured WITHOUT the trained padding newlines, which is how the * live fleet actually emits it ("", * tool_arg_truncation.md log:274). The parser has always read that shape; the * grammar used to swallow it whole. */ static void test_unpadded_close_is_grammatical(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); bool ok; tg_state *tight = feed_ok(g, "\n\n/tmp/x.c", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(tight)); tg_state_free(tight); /* Two parameters, no padding between them either. */ tg_state *two = feed_ok(g, "\n\n*.c" "\nsrc", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(two)); tg_state_free(two); tg_free(g); jr_free(tools); } /* ============================================================ * MISSING-REQUIRED-ARG IMPOSSIBILITY. The empty-VALUE guard above only binds a * key the model DID emit; leaving a required key out entirely used to be a legal * path ("\n" completed with zero parameters), and the * parser turned that into `{}`. Live on the fleet 2026-08-06, WITH the grammar * on, that is exactly what reached the client as `dispatch_agent {}` — refused for a * missing "subtask" at the cost of a wasted round trip. The XML dialect now * enforces `required` the way the JSON dialect always has. * ============================================================ */ /* dispatch_agent(subtask REQUIRED) + check_agents(agent_ids, none required) — * The client's real orchestrator schema, which is where this was observed. */ static const char *REQUIRED_TOOLS_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"dispatch_agent\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"subtask\":{\"type\":\"string\"}},\"required\":[\"subtask\"]}}}," "{\"type\":\"function\",\"function\":{\"name\":\"check_agents\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"agent_ids\":{\"type\":\"array\"}},\"required\":[]}}}]"; static void test_missing_required_arg_rejected(void) { jr_value *tools = jr_parse(REQUIRED_TOOLS_JSON); tg_grammar *g = tg_build(tools); bool ok; /* THE DEFECT: a no-argument call must not be completable. */ tg_state *bare = feed_ok(g, "\n", &ok); ASSERT_FALSE(ok); /* the '/' of "" is masked out */ ASSERT_FALSE(tg_complete(bare)); tg_state_free(bare); /* Supplying the required key completes normally. */ tg_state *full = feed_ok(g, "\n\ndo it\n\n" "", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(full)); tg_state_free(full); /* A tool with an EMPTY `required` list is unaffected: no-arg still closes. */ tg_state *none = feed_ok(g, "\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(none)); tg_state_free(none); /* Same guarantee under forced decoding (tool_choice "required"), which is * the mode the orchestrator actually runs in. */ tg_state *f = tg_state_new_forced(g, NULL); tg_reasoning_end(f); feed_forced(f, "\n", &ok); ASSERT_FALSE(ok); ASSERT_FALSE(tg_complete(f)); tg_state_free(f); /* No schema `required` at all (the pre-existing TOOLS_JSON) keeps the old * permissive behavior — this change only binds what the schema declares. */ jr_value *loose_tools = jr_parse(TOOLS_JSON); tg_grammar *lg = tg_build(loose_tools); tg_state *loose = feed_ok(lg, "\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(loose)); tg_state_free(loose); tg_free(lg); jr_free(loose_tools); tg_free(g); jr_free(tools); } /* Each call in a stream is judged on its own: a satisfied call must not let a * following empty one through (seen_keys resets when a NAME is matched). */ static void test_required_resets_between_calls(void) { jr_value *tools = jr_parse(REQUIRED_TOOLS_JSON); tg_grammar *g = tg_build(tools); bool ok; tg_state *st = feed_ok(g, "\n", &ok); ASSERT_TRUE(ok); /* Start a SECOND call that omits its required key — still rejected. */ char one[2] = {0, 0}; const char *second = "\n\n"; bool second_ok = true; for (const char *p = second; *p; p++) { one[0] = *p; if (!tg_token_allowed(st, one)) { second_ok = false; break; } tg_advance(st, one); } ASSERT_FALSE(second_ok); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * FORCED EARLY-ARM (BUG 1 hardening): the mask must engage the instant the model * starts a tool-call opener DURING reasoning — WITHOUT waiting for "". * ============================================================ */ static void test_forced_early_arm_on_opener(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; /* Reasoning grace: prose unconstrained, mask off. */ feed_forced(st, "thinking about it", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_mask_active(st)); /* Model starts an opener mid-reasoning (no "") — mask arms NOW. */ feed_forced(st, "")); ASSERT_TRUE(tg_token_allowed(st, "read>")); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * FORCED REASONING BUDGET: a model that degenerates in and NEVER emits * "" must still be compelled — after TG_REASON_BUDGET grace chars the * opener is required (prose becomes impossible), so a repetition loop cannot * stream unconstrained forever. * ============================================================ */ static void test_forced_reasoning_budget(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); /* Stream a long degenerate ramble (spaces — allowed in grace) with NO * "" and NO opener, past the internal reasoning budget (8192 chars). * Feed a generous 16384 so we clear the budget regardless. Every space stays * in FREE grace and counts against the budget; mask stays off until it trips. */ char one[2] = {' ', 0}; bool ever_masked = false; for (int i = 0; i < 16384; i++) { if (!tg_token_allowed(st, one)) break; /* prose disallowed once budget trips */ tg_advance(st, one); if (tg_mask_active(st)) { ever_masked = true; break; } } /* The grace is spent: the mask armed (opener compelled) and non-opener prose * is impossible; a valid opener is still accepted. */ ASSERT_TRUE(ever_masked); ASSERT_TRUE(tg_mask_active(st)); ASSERT_FALSE(tg_token_allowed(st, "more prose")); ASSERT_TRUE(tg_token_allowed(st, "")); tg_state_free(st); /* Opener-RICH ramble must ALSO trip the budget. The live degeneration is full * of ""/"")); ASSERT_FALSE(tg_token_allowed(st3, "still rambling")); tg_state_free(st3); tg_free(g); jr_free(tools); } /* ============================================================ * ENUM — the live switch_mode victim, XML dialect. Every declared mode is * still emittable and completes; anything off the list is structurally * impossible. BOTH halves matter: a grammar that rejected every value would * pass the reject half on its own. * ============================================================ */ static void test_enum_closes_switch_mode(void) { jr_value *tools = jr_parse(SWITCH_MODE_JSON); tg_grammar *g = tg_build(tools); ASSERT_NOT_NULL(g); static const char *const MODES[] = { "chat", "plan", "act", "orchestrate" }; for (int i = 0; i < 4; i++) { char call[256]; snprintf(call, sizeof(call), "\n" "\n%s\n\n" "\nthe job needs it\n\n" "", MODES[i]); bool ok; tg_state *st = feed_ok(g, call, &ok); ASSERT_TRUE(ok); /* every legal mode still accepted */ ASSERT_TRUE(tg_complete(st)); tg_state_free(st); } /* The bug this closes. */ ASSERT_TRUE(feed_rejects(g, "\n\nbanana")); /* Off-set values that share a prefix die where they diverge. */ ASSERT_TRUE(feed_rejects(g, "\n\nplans")); /* A strict PREFIX of a member is not a member: the terminator cannot start. */ ASSERT_TRUE(feed_rejects(g, "\n\npla\n")); /* An empty enum VALUE is impossible (no member is empty). */ ASSERT_TRUE(feed_rejects(g, "\n\n\n")); /* A member split across token boundaries still matches. */ tg_state *split = tg_state_new(g); tg_advance(split, "\n\norch"); ASSERT_TRUE(tg_token_allowed(split, "estrate\n\n")); ASSERT_FALSE(tg_token_allowed(split, "id\n\n")); /* "orchid" */ tg_state_free(split); tg_free(g); jr_free(tools); } /* An enum on an OPTIONAL key constrains the value WITHOUT making the key * mandatory: omitting it entirely still completes. */ static void test_enum_on_optional_key(void) { jr_value *tools = jr_parse(ENUM_OPTIONAL_JSON); tg_grammar *g = tg_build(tools); bool ok; tg_state *a = feed_ok(g, "\n\nhi\n\n", &ok); ASSERT_TRUE(ok); /* level omitted → still a complete call */ ASSERT_TRUE(tg_complete(a)); tg_state_free(a); tg_state *b = feed_ok(g, "\n\nhi\n\n" "\nwarn\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(b)); tg_state_free(b); /* Present but off-set is still impossible. */ ASSERT_TRUE(feed_rejects(g, "\n\nhi\n\n" "\nloud")); tg_free(g); jr_free(tools); } /* Enums we cannot represent DEGRADE to a free-text VALUE — they never reject a * legal call (the TG_REQ_MAX_KEYS precedent). */ static void test_enum_degrades_when_unrepresentable(void) { jr_value *tools = jr_parse(ENUM_DEGRADE_JSON); tg_grammar *g = tg_build(tools); ASSERT_NOT_NULL(g); bool ok; /* nums / mixed / blank / nolist / clash: all dropped → free text, so text * that is nowhere in the enum is accepted rather than rejected. */ static const char *const DROPPED[] = { "nums", "mixed", "blank", "nolist", "clash" }; for (int i = 0; i < 5; i++) { char call[256]; snprintf(call, sizeof(call), "\n\nanything at all\n\n" "", DROPPED[i]); tg_state *st = feed_ok(g, call, &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); } /* notype: string members and NO declared type → the enum DOES apply. */ tg_state *d = feed_ok(g, "\n\nyes\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(d)); tg_state_free(d); ASSERT_TRUE(feed_rejects(g, "\n\nmaybe")); tg_free(g); jr_free(tools); } /* An enum larger than the cap degrades to free text rather than rejecting. */ static void test_enum_over_cap_degrades(void) { char schema[8192]; int n = snprintf(schema, sizeof(schema), "[{\"type\":\"function\",\"function\":{\"name\":\"pick\"," "\"parameters\":{\"type\":\"object\",\"properties\":{" "\"v\":{\"type\":\"string\",\"enum\":["); for (int i = 0; i < 65; i++) /* 65 > TG_ENUM_MAX_MEMBERS */ n += snprintf(schema + n, sizeof(schema) - (size_t)n, "%s\"m%03d\"", i ? "," : "", i); snprintf(schema + n, sizeof(schema) - (size_t)n, "]}},\"required\":[\"v\"]}}}]"); jr_value *tools = jr_parse(schema); tg_grammar *g = tg_build(tools); ASSERT_NOT_NULL(g); bool ok; tg_state *st = feed_ok(g, "\n\nanything\n\n", &ok); ASSERT_TRUE(ok); /* off-list accepted → the enum was dropped */ ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * LANDING WINDOW (tg_reasoning_end_injected) — the frame-capture cause fix. * * When a turn spends its reasoning_budget the SERVE force-decodes "" * itself. The grammar cannot tell that close from the model's own, so it used * to compel the tool-call opener from the very next char — while the model was * still mid-sentence. The model then emitted the compelled * "\n\n" and dumped the rest of its thought, * often a COMPLETE tool call, into the free-text VALUE. Recorded live: * a recorded native-XML capture and three recorded Hermes-shape captures. * * tg_reasoning_end_injected grants a bounded AUTO-like grace so the model can * land: finish its sentence, emit its own close, and open the call itself. * ============================================================ */ /* The prose the model actually wrote after being cut off (the Hermes lead-in * capture, the text that was captured into the `path` argument). */ static const char CAPTURED_LEADIN_PROSE[] = "\n\nLet me look at the Makefile to understand the build structure:\n\n"; /* After an INJECTED close the model's unfinished sentence must still be legal. * This is the assertion the bug fails: pre-fix, the first prose char is * off-grammar and the model is forced to open a frame around its thought. */ static void test_landing_window_lets_the_model_finish(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); /* The serve force-decoded "" and told the grammar it was injected. */ tg_reasoning_end_injected(st); ASSERT_INT_EQ(tg_debug_landing_chars(st), 512); /* AUTO-like during the window: no mask, prose allowed. */ ASSERT_FALSE(tg_mask_active(st)); ASSERT_TRUE(tg_token_allowed(st, "Let me look at the Makefile")); bool ok; feed_forced(st, CAPTURED_LEADIN_PROSE, &ok); ASSERT_TRUE(ok); /* the whole sentence stayed legal */ ASSERT_FALSE(tg_in_call(st)); /* and was NOT swallowed into a frame */ tg_state_free(st); tg_free(g); jr_free(tools); } /* End to end on the real capture: after the injected close the model writes its * prose AND THEN its own call — which now lands as a real call instead of as the * string value of a compelled `read`. The path is the one the model asked for. */ static void test_landing_window_recovers_the_real_capture(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; tg_reasoning_end_injected(st); feed_forced(st, CAPTURED_LEADIN_PROSE, &ok); ASSERT_TRUE(ok); /* The model now opens the call it meant to make all along. */ feed_forced(st, "\n\n\n" "lumen/Makefile\n\n\n" "", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); /* a COMPLETE call, not a captured one */ tg_state_free(st); tg_free(g); jr_free(tools); } /* The window ends the moment the model emits its OWN "" — it has now * genuinely concluded, so the opener is compelled again immediately. This is * what actually terminates the window in every recorded capture (all of them * open with the model's own close). */ static void test_landing_window_ends_on_models_own_close(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; tg_reasoning_end_injected(st); ASSERT_INT_EQ(tg_debug_landing_chars(st), 512); /* Exactly the shape seen live: the model writes the close it had not * reached yet as the first thing after being cut off. */ feed_forced(st, "", &ok); ASSERT_TRUE(ok); ASSERT_INT_EQ(tg_debug_landing_chars(st), 0); /* window closed early */ /* Compelled again: prose is off-grammar, an opener is not. */ ASSERT_FALSE(tg_token_allowed(st, "Let me look")); ASSERT_TRUE(tg_token_allowed(st, "" in the stream exactly as before * and compels the opener from the next char. Byte-identical to pre-fix * behavior, which is the bar: only an INJECTED close changes anything. */ static void test_landing_window_absent_on_self_close(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); bool ok; feed_forced(st, "thinking it through", &ok); ASSERT_TRUE(ok); ASSERT_INT_EQ(tg_debug_landing_chars(st), 0); /* no grace granted */ ASSERT_TRUE(tg_forced_reasoning_ended(st)); ASSERT_FALSE(tg_token_allowed(st, "prose")); /* compelled immediately */ ASSERT_TRUE(tg_token_allowed(st, "")); ASSERT_TRUE(tg_token_allowed(st, "read>")); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * Build guards: empty / non-array tools → NULL grammar. * ============================================================ */ static void test_build_guards(void) { jr_value *empty = jr_parse("[]"); ASSERT_NULL(tg_build(empty)); jr_free(empty); jr_value *obj = jr_parse("{\"x\":1}"); ASSERT_NULL(tg_build(obj)); jr_free(obj); } /* ============================================================ * THE NEAR-MISS OPENER — measured on qwen36-bf16 (BF16 full weights, 7 tools, * 558-token prompt) with the grammar OFF. The model does not drift to JSON and * does not answer in prose: it invents a tool-call frame that is one character * away from the real opener. * * \n\nbash… * * This is the case the AUTO/FORCED split actually turns on, and it is NOT the * "grammar armed, then went DEAD" story. In AUTO the mask is inactive for the * whole of TG_FREE (tg_mask_active only engages at tg_in_call), so every one of * those characters is sampled unconstrained and the walk simply never enters the * skeleton — an armed AUTO grammar would have changed nothing about this turn. * FORCED engages the moment an opener prefix starts (free_open > 0), so the 's' * that breaks "\n\n", four of * them in one fleet log). There the opener DOES complete, the walk enters * the skeleton, and AUTO masks "" out — so the two sub-classes have * opposite outcomes under the same mode, and only FORCED covers both. */ tg_state *inner = tg_state_new(g); tg_advance(inner, ""); ASSERT_TRUE(tg_in_call(inner)); ASSERT_TRUE(tg_mask_active(inner)); ASSERT_FALSE(tg_token_allowed(inner, "\n")); /* the invented tag */ ASSERT_TRUE(tg_token_allowed(inner, "\n")); tg_state_free(inner); /* FORCED, same bytes: the mask arms on '<' and the 's' is off-grammar. */ tg_state *fs = tg_state_new_forced(g, NULL); tg_reasoning_end(fs); tg_advance(fs, "")); /* the real opener still is */ tg_state_free(fs); tg_free(g); jr_free(tools); } /* ============================================================ * DEAD IS NOT FREE. tg_in_call() is false for both, which is why "the AUTO * grammar switched itself off" was unobservable: a walk that went off-grammar * mid-call and a walk that never started one reported the same thing. A DEAD * walk also reports its mask INACTIVE, so every remaining token of the turn is * sampled with no constraint at all. * ============================================================ */ static void test_dead_walk_is_distinguishable_from_free(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); /* FREE: prose, no call. Not dead. */ tg_state *free_st = tg_state_new(g); tg_advance(free_st, "The answer is 42."); ASSERT_FALSE(tg_in_call(free_st)); ASSERT_FALSE(tg_is_dead(free_st)); tg_state_free(free_st); /* DEAD: a real opener, then an off-grammar byte inside the skeleton. */ tg_state *dead_st = tg_state_new(g); tg_advance(dead_st, ""); ASSERT_TRUE(tg_in_call(dead_st)); ASSERT_TRUE(tg_mask_active(dead_st)); tg_advance(dead_st, "\nnot-a-parameter"); ASSERT_TRUE(tg_is_dead(dead_st)); ASSERT_FALSE(tg_in_call(dead_st)); /* indistinguishable from FREE here */ ASSERT_FALSE(tg_mask_active(dead_st));/* the constraint is off for the rest */ tg_state_free(dead_st); tg_free(g); jr_free(tools); } /* ============================================================ * CONTAINER-TYPED VALUES: the frame was constrained, the ARGUMENT was not. * * The client's REAL todowrite schema (the client's schema code emit_todo_tool): * `todos` is {"type":"array","items":{"type":"object",…}}, the only tool in the * live toolset with a nested shape. `read` and `bash` — flat, single-string * arguments — were flawless for a whole run; todowrite failed three times with * the same malformation and never recovered, because the bytes inside * were free text. * * tool_calling.c splices a container-typed value into the call's `arguments` * object RAW, so one malformed value does not produce a bad argument — it * produces an `arguments` string that will not parse at all, taking every other * argument of the call with it. * ============================================================ */ static const char *TODOWRITE_JSON = "[{\"type\":\"function\",\"function\":{\"name\":\"todowrite\"," "\"parameters\":{\"type\":\"object\",\"properties\":" "{\"todos\":{\"type\":\"array\",\"items\":{\"type\":\"object\"," "\"properties\":{\"content\":{\"type\":\"string\",\"minLength\":1}," "\"status\":{\"type\":\"string\"," "\"enum\":[\"pending\",\"in_progress\",\"completed\"]}}," "\"required\":[\"content\",\"status\"]}}}," "\"required\":[\"todos\"]}}}]"; /* Wrap a `todos` VALUE in the trained XML call the model actually emits. */ static char *todowrite_call(const char *value) { static char buf[2048]; snprintf(buf, sizeof(buf), "\n\n%s\n\n" "", value); return buf; } static void test_malformed_nested_argument_is_rejected(void) { jr_value *tools = jr_parse(TODOWRITE_JSON); tg_grammar *g = tg_build(tools); ASSERT_NOT_NULL(g); /* The live failures, verbatim in shape: a bare value where a key was owed. */ ASSERT_TRUE(feed_rejects(g, todowrite_call( "[{\"content\":\"Remove idx_points_scope\",\"status\":\"in_progress\"}," "{\"content\":\"\",\"\"}]"))); ASSERT_TRUE(feed_rejects(g, todowrite_call( "[{\"content\":\"a\",\"status\":\"pending\"}," "{\"content\":\"_verify the fix works end-to-end_\",\"_verify\"}]"))); /* What the model meant is still reachable, end to end. */ bool ok = false; tg_state *st = feed_ok(g, todowrite_call( "[{\"content\":\"Remove idx_points_scope\",\"status\":\"in_progress\"}," "{\"content\":\"verify the fix works\",\"status\":\"pending\"}]"), &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); /* An empty list is a legal, reachable emission — the container-typed rule * bans "" (a string, wrong type), not emptiness. */ ok = false; tg_state *st2 = feed_ok(g, todowrite_call("[]"), &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st2)); tg_state_free(st2); /* Wrong JSON type for an "array" parameter: well-formed, still refused. */ ASSERT_TRUE(feed_rejects(g, todowrite_call("{\"a\":1}"))); ASSERT_TRUE(feed_rejects(g, todowrite_call("\"buy milk\""))); /* Unclosed: "" is not a legal continuation until the value closes. */ ASSERT_TRUE(feed_rejects(g, todowrite_call("[{\"content\":\"a\""))); tg_free(g); jr_free(tools); } /* The grammar and the parser must not disagree about where a value ended. * parse_call_arguments finds the value's end with strstr(v, ""), so * those bytes INSIDE a JSON string would cut the value mid-structure and hand * write_typed_value an un-parseable prefix. Off-grammar, therefore never drawn. */ static void test_terminator_inside_a_json_string_is_off_grammar(void) { jr_value *tools = jr_parse(TODOWRITE_JSON); tg_grammar *g = tg_build(tools); ASSERT_TRUE(feed_rejects(g, todowrite_call( "[{\"content\":\"ab\",\"status\":\"pending\"}]"))); /* A lone '<' inside a string is ordinary text and must stay legal — the * guard is the whole literal, not the character. */ bool ok = false; tg_state *st = feed_ok(g, todowrite_call( "[{\"content\":\"a < b\",\"status\":\"pending\"}]"), &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* Scope guard: a STRING parameter keeps its free-text VALUE. `write`'s * `content` and `bash`'s `command` carry braces, brackets and quotes all day — * constraining them would break every file the model writes. */ static void test_string_parameters_stay_free_text(void) { jr_value *tools = jr_parse(TOOLS_JSON); /* glob/read: all string params */ tg_grammar *g = tg_build(tools); bool ok = false; tg_state *st = feed_ok(g, "\n\n{ not [ json \" at all\n" "\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* ============================================================ * THE BUDGET MUST TRIP AT A TOKEN BOUNDARY, NOT MID-TOKEN. * * Live regression, qwen36-bf16, one recorded session, 2026-08-26: THREE * turns ended * * [tool_grammar] turn end: the walk went DEAD in FORCED mode — the mask * reported inactive from that point and the rest of this turn decoded * UNCONSTRAINED (dead_turns=1..3) * [tool_grammar] turn mask density: masked_steps=0 sparse_rows=0 * * masked_steps=0 is the whole story: the walk died BEFORE the mask was ever * evaluated, so nothing the model wrote for the rest of that turn was * constrained. And because tg_forced_reasoning_ended() kept saying "reasoning * is over" from a walk that had already given up, the serve flipped the emitter * to the CONTENT channel (serve_generation.c) — so the rest of the model's think * block was recorded as an ANSWER, mid-sentence. * * The cause is a granularity mismatch this suite could not see. The budget trip * consumes its char and returns, on the stated promise "arm cleanly, then mask * the next step". That promise holds only if the next CHAR is the next STEP. * It is not: tg_advance walks a whole token, so the next char of the SAME token * is judged under `forced` with no mask in between, and one prose char there is * TG_FREE_BREAK → TG_DEAD. * * Every pre-existing budget test fed the grammar ONE CHAR AT A TIME, which is * the one granularity at which the bug cannot occur. These feed tokens. * ============================================================ */ static void test_reason_budget_trips_mid_token_without_dying(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); /* Realistic BPE pieces: multi-char, prose, no opener, no "". The * budget (8192 chars) cannot land on a token boundary with a 9-char piece, * so the trip is always mid-token — exactly the live shape. */ const char *piece = " thinking"; ASSERT_TRUE(strlen(piece) > 1); bool armed = false; for (int i = 0; i < 4096 && !armed; i++) { tg_advance(st, piece); armed = tg_mask_active(st); if (tg_is_dead(st)) break; } /* THE RED ASSERTION. Before the fix the walk is DEAD here and `armed` never * became true — mask inactive for the rest of the turn, masked_steps=0. */ ASSERT_FALSE(tg_is_dead(st)); ASSERT_TRUE(armed); ASSERT_TRUE(tg_mask_active(st)); /* And the compelled region really is compelled from the next token on. */ ASSERT_TRUE(tg_token_allowed(st, "")); ASSERT_FALSE(tg_token_allowed(st, " still rambling")); tg_state_free(st); tg_free(g); jr_free(tools); } /* The same hazard at the OTHER end of the landing window: the window expires * mid-token, and the remaining chars of that token must not be judged under the * compelled rule either. */ static void test_landing_window_expires_mid_token_without_dying(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); tg_reasoning_end_injected(st); const char *piece = " landing"; /* 8 chars; 512 % 8 == 0 would boundary-align */ const char *odd = " land"; /* 5 chars; 512 % 5 != 0 → expires mid-token */ (void)piece; for (int i = 0; i < 512 && tg_debug_landing_chars(st) > 0; i++) { tg_advance(st, odd); if (tg_is_dead(st)) break; } ASSERT_FALSE(tg_is_dead(st)); ASSERT_INT_EQ(tg_debug_landing_chars(st), 0); ASSERT_TRUE(tg_mask_active(st)); ASSERT_TRUE(tg_token_allowed(st, "")); tg_state_free(st); tg_free(g); jr_free(tools); } /* A DEAD walk has no standing to tell the serve that reasoning is over. * * sg_tg_forced_ended() (serve_generation.c) drives exactly one thing: it flips * the emitter out of its channel so a COMPELLED call is parsed as a tool * call instead of landing in reasoning_content. A dead walk compels nothing — * its mask is inactive and every remaining token is unconstrained — so answering * "yes" there routes the model's ongoing THOUGHT into the content channel and * the transcript records it as an answer. That is the leak. * * Everything else in this module already refuses to speak for a dead walk * (tg_mask_active, and the serve's own tool-call verdict); this makes the * reasoning-ended report consistent with them. */ static void test_dead_walk_does_not_report_reasoning_ended(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_forced(g, NULL); tg_reasoning_end(st); ASSERT_TRUE(tg_forced_reasoning_ended(st)); /* alive + compelling: yes */ tg_advance(st, "prose"); /* off-grammar while compelled */ ASSERT_TRUE(tg_is_dead(st)); ASSERT_FALSE(tg_mask_active(st)); /* the constraint is off … */ ASSERT_FALSE(tg_forced_reasoning_ended(st)); /* … so this must be off too */ tg_state_free(st); tg_free(g); jr_free(tools); } /* The sampler's full mask rejects a token whose first 1 or 2 bytes kill the walk * without walking the rest (engine.c grammar_first_char_verdicts) — sound only * because DEAD is absorbing. Checked at every prefix of a forced XML call. */ /* ============================================================ * tool_choice "required" with thinking ON (tg_state_new_required, think_open): * the think block is NEVER judged — a "" or "" the call is compelled. * ============================================================ */ static void test_required_think_gated_reasoning_is_free(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_required(g, true); ASSERT_NOT_NULL(st); ASSERT_FALSE(tg_mask_active(st)); const char *thought = "I could emit \n\n\nx\n " "or -- let me think about \n\nSure"); ASSERT_FALSE(tg_is_dead(st)); ASSERT_TRUE(tg_mask_active(st)); ASSERT_TRUE(tg_forced_reasoning_ended(st)); ASSERT_FALSE(tg_token_allowed(st, "Let me see the rest of the file.")); ASSERT_FALSE(tg_token_allowed(st, "{\"name\": \"read\"")); feed_forced(st, "\n\n\n\nsrc/main.c\n" "\n\n", &ok); ASSERT_TRUE(ok); ASSERT_TRUE(tg_complete(st)); ASSERT_FALSE(tg_mask_active(st)); tg_state_free(st); tg_free(g); jr_free(tools); } /* "required" with NO open think block (thinking off): compelled from token 0. */ static void test_required_without_think_compels_from_start(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_required(g, false); ASSERT_TRUE(tg_mask_active(st)); ASSERT_FALSE(tg_token_allowed(st, "Sure, let me look.")); ASSERT_TRUE(tg_token_allowed(st, "\n")); ASSERT_TRUE(tg_token_allowed(st, "")); tg_state_free(st); tg_free(g); jr_free(tools); } /* The serve's reasoning_budget force-close: the injected "" ends the * think-gated block and grants the landing window; the model's own close then * compels the call. */ static void test_required_injected_close_lands(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); tg_state *st = tg_state_new_required(g, true); bool ok; feed_forced(st, "still thinking about "); /* the serve's injected close */ tg_reasoning_end_injected(st); ASSERT_FALSE(tg_mask_active(st)); /* landing: the sentence may finish */ feed_forced(st, " and then I read it.", &ok); ASSERT_TRUE(ok); feed_forced(st, "", &ok); /* the model's own close */ ASSERT_TRUE(ok); ASSERT_TRUE(tg_mask_active(st)); ASSERT_FALSE(tg_token_allowed(st, "prose")); tg_state_free(st); tg_free(g); jr_free(tools); } /* Parse `text` as the serve would (Qwen XML dialect) and return the one call's * "name:arguments", or "" when it does not yield exactly one call. */ static void parse_one_call(const char *text, const jr_value *tools, char *out, size_t cap) { tc_tool_call *calls = NULL; int n = 0; out[0] = '\0'; if (tc_parse_tool_calls(text, tools, TC_DIALECT_QWEN_XML, &calls, &n) == 0 && n == 1) snprintf(out, cap, "%s:%s", calls[0].name, calls[0].arguments); tc_tool_calls_free(calls, n); } /* A "required" turn may end only where that is legal: never before the call * opens, and inside the call only where the serve can close it — and the * closed call parses to what the model wrote. */ static void test_required_end_and_closing_text(void) { jr_value *tools = jr_parse(CLIENT_FLOORED_JSON); tg_grammar *g = tg_build(tools); char closing[TG_CLOSING_MAX], text[512], got[512]; bool ok; tg_state *st = tg_state_new_required(g, true); ASSERT_TRUE(tg_end_allowed(st)); /* thinking: free */ feed_forced(st, "plan\n\n", &ok); ASSERT_FALSE(tg_end_allowed(st)); /* compelled, no call yet */ ASSERT_INT_EQ(tg_closing_text(st, closing, TG_CLOSING_MAX), -1); feed_forced(st, "\n\n\n", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_end_allowed(st)); /* floored value still empty */ feed_forced(st, "ls -la", &ok); ASSERT_TRUE(tg_end_allowed(st)); ASSERT_INT_EQ(tg_closing_text(st, closing, TG_CLOSING_MAX), (int)strlen("\n\n\n")); ASSERT_STR_EQ(closing, "\n\n\n"); tg_advance(st, closing); ASSERT_TRUE(tg_complete(st)); ASSERT_TRUE(tg_end_allowed(st)); /* done: free again */ tg_state_free(st); /* The closed call is the call the parser reads from the unclosed text. */ snprintf(text, sizeof(text), "\n\n\nls -la%s", "\n\n\n"); parse_one_call(text, tools, got, sizeof(got)); ASSERT_STR_EQ(got, "bash:{\"command\":\"ls -la\"}"); parse_one_call("\n\n\nls -la", tools, text, sizeof(text)); ASSERT_STR_EQ(text, got); /* Part-way through "": the rest of the literal, then the wrapper. */ st = tg_state_new_required(g, false); feed_forced(st, "\n\n\nls\n\n" "\n")); ASSERT_STR_EQ(closing, "tion>\n"); tg_state_free(st); /* Bare form (no wrapper): closed without "". */ st = tg_state_new_required(g, false); feed_forced(st, "\n\npwd", &ok); ASSERT_INT_EQ(tg_closing_text(st, closing, TG_CLOSING_MAX), (int)strlen("\n\n")); ASSERT_STR_EQ(closing, "\n\n"); tg_state_free(st); /* A required key still missing: no closer, the turn may not end. */ st = tg_state_new_required(g, false); feed_forced(st, "\n\n\na.c", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_end_allowed(st)); feed_forced(st, "\n\n", &ok); ASSERT_FALSE(tg_end_allowed(st)); tg_state_free(st); tg_free(g); jr_free(tools); /* An enum value is closable only once it is a whole member. */ tools = jr_parse(SWITCH_MODE_JSON); g = tg_build(tools); st = tg_state_new_required(g, false); feed_forced(st, "\n\n\nr\n" "\n\npl", &ok); ASSERT_TRUE(ok); ASSERT_FALSE(tg_end_allowed(st)); feed_forced(st, "an", &ok); ASSERT_TRUE(tg_end_allowed(st)); ASSERT_INT_EQ(tg_closing_text(st, closing, TG_CLOSING_MAX), (int)strlen("\n\n\n")); tg_advance(st, closing); ASSERT_TRUE(tg_complete(st)); tg_state_free(st); tg_free(g); jr_free(tools); } static void test_first_char_prefilter_is_sound(void) { jr_value *tools = jr_parse(TOOLS_JSON); tg_grammar *g = tg_build(tools); static const char *TOKENS[] = { "", "<", "\n", "", "", "\n", " ", "Sure", "", "src/main.c", "{\"", "", "", "\u00e9", NULL }; const char *script = "let me think\n\n\n" "\nsrc/main.c\n\n\n"; char one[2] = { 0, 0 }; int checked = 0, violations = 0; /* The legacy forced walk and the think-gated "required" walk. */ for (int w = 0; w < 2; w++) { tg_state *st = w ? tg_state_new_required(g, true) : tg_state_new_forced(g, NULL); for (const char *p = script; ; p++) { for (int t = 0; TOKENS[t]; t++) { char two[3] = { TOKENS[t][0], TOKENS[t][0] ? TOKENS[t][1] : 0, 0 }; one[0] = TOKENS[t][0]; if (tg_token_allowed(st, TOKENS[t]) && (!tg_token_allowed(st, one) || !tg_token_allowed(st, two))) violations++; checked++; } if (!*p) break; one[0] = *p; tg_advance(st, one); } ASSERT_TRUE(tg_complete(st)); tg_state_free(st); } ASSERT_INT_EQ(violations, 0); ASSERT_TRUE(checked > 2000); tg_free(g); jr_free(tools); } int main(void) { printf("\n=== tool_grammar (grammar-constrained tool decoding) ===\n\n"); RUN_TEST(wrapped_accepts); RUN_TEST(first_char_prefilter_is_sound); RUN_TEST(bare_accepts); RUN_TEST(prose_then_call); RUN_TEST(pure_prose_no_call); RUN_TEST(json_form_rejected); RUN_TEST(unknown_function_rejected); RUN_TEST(unknown_param_rejected); RUN_TEST(cross_token_name); RUN_TEST(multi_param); RUN_TEST(forced_rejects_prose_and_json); RUN_TEST(forced_accepts_call); RUN_TEST(forced_reasoning_then_call); RUN_TEST(forced_function_pins_name); RUN_TEST(forced_self_detects_think_close); RUN_TEST(forced_think_close_split_tokens); RUN_TEST(forced_direct_call_no_think); RUN_TEST(auto_mode_unchanged); RUN_TEST(auto_never_masks_a_near_miss_opener); RUN_TEST(dead_walk_is_distinguishable_from_free); RUN_TEST(empty_value_closes_the_parameter); RUN_TEST(empty_value_off_grammar_when_floored); RUN_TEST(unfloored_value_may_still_be_empty); RUN_TEST(unpadded_close_is_grammatical); RUN_TEST(missing_required_arg_rejected); RUN_TEST(required_resets_between_calls); RUN_TEST(forced_early_arm_on_opener); RUN_TEST(forced_reasoning_budget); RUN_TEST(reason_budget_trips_mid_token_without_dying); RUN_TEST(landing_window_expires_mid_token_without_dying); RUN_TEST(dead_walk_does_not_report_reasoning_ended); RUN_TEST(landing_window_lets_the_model_finish); RUN_TEST(landing_window_recovers_the_real_capture); RUN_TEST(landing_window_ends_on_models_own_close); RUN_TEST(landing_window_closes_at_budget); RUN_TEST(landing_window_absent_on_self_close); RUN_TEST(landing_window_absent_in_auto_mode); RUN_TEST(landing_window_opener_still_engages); RUN_TEST(enum_closes_switch_mode); RUN_TEST(enum_on_optional_key); RUN_TEST(enum_degrades_when_unrepresentable); RUN_TEST(enum_over_cap_degrades); RUN_TEST(malformed_nested_argument_is_rejected); RUN_TEST(terminator_inside_a_json_string_is_off_grammar); RUN_TEST(string_parameters_stay_free_text); RUN_TEST(build_guards); RUN_TEST(required_think_gated_reasoning_is_free); RUN_TEST(required_without_think_compels_from_start); RUN_TEST(required_injected_close_lands); RUN_TEST(required_end_and_closing_text); TEST_REPORT(); return __ft_failed == 0 ? 0 : 1; }