fix: release expression token storage

docs/CURRENT_STATUS.md

@@ -27,7 +27,9 @@ Verified in this checkout on 2026-09-22:
 - `./claro doctor`: all checks `OK`.
 - `./claro validate`: validation passed.
 
-The `RUN COMMAND` path now decodes POSIX `pclose()` wait status before storing `LASTEXIT`, so a child that exits with code 3 exposes `3` rather than the encoded status `768`. Focused coverage is `tests/42_last_exit_code.claro`. This slice does not yet fix memory cleanup, HTTP buffering/status handling, or other review findings. Claro remains a trusted-script interpreter, not a sandbox.
+This run's narrow memory slice releases expression token arrays and token strings after every `eval_expr()` call. Focused coverage is `tools/validate_memory_cleanup.py`; it reports `PASS: expression token allocations are released` under an ASan/UBSan build with LeakSanitizer checking. Remaining memory-growth areas include runtime-owned overwritten values, loaded program storage, and other expression temporaries. The `RUN COMMAND` path remains trusted shell execution, not a sandbox.
+
+The `RUN COMMAND` path now decodes POSIX `pclose()` wait status before storing `LASTEXIT`, so a child that exits with code 3 exposes `3` rather than the encoded status `768`. Focused coverage is `tests/42_last_exit_code.claro`. This slice does not yet fix HTTP buffering/status handling or the remaining memory-growth boundaries. Claro remains a trusted-script interpreter, not a sandbox.
 
 ## Feature matrix
 

docs/HARDENING.md

@@ -8,3 +8,15 @@ Safety caps (to prevent memory abuse):
 - Maximum program lines: 500,000
 
 If a file exceeds these limits, the loader fails cleanly.
+
+## Expression-token cleanup
+
+Each expression now releases its token strings and token-array storage before returning. This is a narrow cleanup boundary; runtime-owned variables, loaded programs, and other allocations remain separate follow-up work.
+
+Focused verification:
+
+```text
+python3 tools/validate_memory_cleanup.py
+```
+
+The validator builds an AddressSanitizer/UndefinedBehaviorSanitizer binary, runs a repeated variable-overwrite probe, and confirms LeakSanitizer no longer reports allocations from `tokenize`/`toks_add`. The interpreter remains a trusted-script runtime, not a sandbox.

src/claro.c

@@ -162,6 +162,7 @@ static Tokens tokenize(const char *s){ Tokens ts; size_t i=0; memset(&ts,0,sizeo
   { size_t st=i; while(s[i]&&!isspace((unsigned char)s[i])&&!strchr("(),+-*/=<>!",s[i])) i++; toks_add(&ts,s+st,i-st); }
  }
  return ts; }
+static void tokens_free(Tokens *ts){ int i; if(!ts) return; for(i=0;i<ts->n;i++) free(ts->t[i]); free(ts->t); ts->t=NULL; ts->n=ts->cap=ts->pos=0; }
 static int tok_end(Tokens *ts){ return ts->pos>=ts->n; }
 static char *tok_peek(Tokens *ts){ return tok_end(ts)?NULL:ts->t[ts->pos]; }
 static char *tok_next(Tokens *ts){ return tok_end(ts)?NULL:ts->t[ts->pos++]; }
@@ -179,7 +180,7 @@ static Value parse_add(Runtime *rt,Tokens *ts){ Value v=parse_mul(rt,ts); while(
 static Value parse_cmp(Runtime *rt,Tokens *ts){ Value v=parse_add(rt,ts); while(!tok_end(ts)){ char *op=tok_peek(ts); int c; if(!(ci_eq(op,"IS")||!strcmp(op,"=")||!strcmp(op,"!=")||!strcmp(op,"<")||!strcmp(op,"<=")||!strcmp(op,">")||!strcmp(op,">="))) break; tok_next(ts); Value r=parse_add(rt,ts); c=value_compare(v,r); if(ci_eq(op,"IS")||!strcmp(op,"=")) v=v_bool(c==0); else if(!strcmp(op,"!=")) v=v_bool(c!=0); else if(!strcmp(op,"<")) v=v_bool(c<0); else if(!strcmp(op,"<=")) v=v_bool(c<=0); else if(!strcmp(op,">")) v=v_bool(c>0); else v=v_bool(c>=0); } return v; }
 static Value parse_and(Runtime *rt,Tokens *ts){ Value v=parse_cmp(rt,ts); while(tok_match(ts,"AND")){ Value r=parse_cmp(rt,ts); v=v_bool(v_truth(v)&&v_truth(r)); } return v; }
 static Value parse_expr(Runtime *rt,Tokens *ts){ Value v=parse_and(rt,ts); while(tok_match(ts,"OR")){ Value r=parse_and(rt,ts); v=v_bool(v_truth(v)||v_truth(r)); } return v; }
-static Value eval_expr(Runtime *rt,const char *expr){ Tokens ts=tokenize(expr); Value v=parse_expr(rt,&ts); return v; }
+static Value eval_expr(Runtime *rt,const char *expr){ Tokens ts=tokenize(expr); Value v=parse_expr(rt,&ts); tokens_free(&ts); return v; }
 
 /* Better string token parsing helpers for commands. */
 static char *unquote_token(const char **ps){ const char *s=*ps; Str b; str_init(&b); while(*s&&isspace((unsigned char)*s)) s++; if(*s=='"'){ s++; while(*s&&*s!='"'){ if(*s=='\\'){ s++; if(*s=='n') str_ch(&b,'\n'); else if(*s=='t') str_ch(&b,'\t'); else if(*s=='r') str_ch(&b,'\r'); else if(*s=='"') str_ch(&b,'"'); else if(*s=='\\') str_ch(&b,'\\'); else if(*s) str_ch(&b,*s); if(*s) s++; } else str_ch(&b,*s++); } if(*s=='"') s++; } else { while(*s&&!isspace((unsigned char)*s)) str_ch(&b,*s++); } *ps=s; return str_take(&b); }

tools/validate_memory_cleanup.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Focused regression check for expression-token cleanup under LeakSanitizer."""
+from pathlib import Path
+import os
+import subprocess
+import sys
+import tempfile
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def main() -> int:
+    if os.name == "nt":
+        print("SKIP: LeakSanitizer check is POSIX-only")
+        return 0
+    with tempfile.TemporaryDirectory(prefix="claro-memory-") as tmp:
+        tmp_path = Path(tmp)
+        binary = tmp_path / "claro-lsan"
+        script = tmp_path / "overwrite.claro"
+        script.write_text(
+            'SET value TO "first"\n'
+            'SET value TO "second"\n'
+            'SAY value\n',
+            encoding="utf-8",
+        )
+        build = subprocess.run(
+            [
+                "gcc", "-std=c99", "-O0", "-g",
+                "-fsanitize=address,undefined",
+                "src/claro.c", "-o", str(binary), "-lm",
+            ], cwd=ROOT, text=True, capture_output=True,
+        )
+        if build.returncode:
+            print(build.stdout, end="")
+            print(build.stderr, end="", file=sys.stderr)
+            return build.returncode
+        run = subprocess.run(
+            [str(binary), str(script)], cwd=ROOT, text=True,
+            capture_output=True,
+            env={**os.environ, "ASAN_OPTIONS": "detect_leaks=1"},
+        )
+        if "toks_add" in run.stderr or "tokenize" in run.stderr:
+            print("FAIL: expression token allocations still leak")
+            print(run.stderr, end="", file=sys.stderr)
+            return 1
+        functional = subprocess.run(
+            [str(binary), str(script)], cwd=ROOT, text=True,
+            capture_output=True,
+            env={**os.environ, "ASAN_OPTIONS": "detect_leaks=0"},
+        )
+        if functional.returncode or "second\n" not in functional.stdout:
+            print("FAIL: overwrite cleanup probe produced the wrong output")
+            print(functional.stdout, end="", file=sys.stderr)
+            return 1
+        print("PASS: expression token allocations are released")
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())