From 4a232501e7aa5d2fb25c071cdcef87bdc2856379 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 6 Sep 2026 22:18:31 +0000 Subject: [PATCH] test: harden CI workflow gate detection --- CHANGELOG.md | 5 ++++ docs/CURRENT_STATUS.md | 2 +- docs/ROADMAP.md | 2 +- tests/test_validate_ci_workflow.py | 46 ++++++++++++++++++++++++++++++ tools/validate_ci_workflow.py | 32 ++++++++++++++++++++- 5 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 tests/test_validate_ci_workflow.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 59d7639..1fbf92c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +### CI release-gate validator hardening + +- Added focused regression coverage for the workflow validator. +- `tools/validate_ci_workflow.py` now checks release commands inside actual `run` steps, so commented-out commands cannot make CI coverage appear complete. + ### Object-alias field CHECK TYPE diagnostics - Added focused typecheck coverage for `CHECK TYPE alias.field IS TYPE` after a simple object alias. diff --git a/docs/CURRENT_STATUS.md b/docs/CURRENT_STATUS.md index 6ed0a2a..7a7afbe 100644 --- a/docs/CURRENT_STATUS.md +++ b/docs/CURRENT_STATUS.md @@ -173,7 +173,7 @@ Ready now: - metadata JSON - completion list - diagnostics helper -- Forgejo/Gitea CI coverage for the current release validation gates (`claro validate`, typecheck diagnostics validation, version convention validation, package security validation, and the CI workflow coverage check) +- Forgejo/Gitea CI coverage for the current release validation gates (`claro validate`, typecheck diagnostics validation, version convention validation, package security validation, and the CI workflow coverage check); the workflow validator verifies these commands occur in executable `run` steps rather than comments Still needed: - syntax highlighting package diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index dea184a..d6be4c7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -31,7 +31,7 @@ See `CURRENT_STATUS.md` for the detailed feature matrix. 1. Keep beginner-facing docs current and separate from historical release notes. 2. Keep examples aligned with the modern simple syntax (`END`, `DO`, short `SET`, short `ASK`) while documenting older compatibility forms separately. -3. Expand validation around typecheck diagnostics and package/networking safety. Current Forgejo/Gitea CI runs the documented release gates (`claro validate`, typecheck diagnostics validation, version convention validation, package security validation, and CI workflow coverage validation). Current package safety validation covers safe project creation, rejecting unsafe project names during `claro new`, safe manifest/lockfile creation, rejecting unsafe package names during `package add`, detecting unsafe package names already present in `claro.project` during `package doctor`, refusing to write lockfile data for unsafe package names during `package lock`, making `package list` fail instead of displaying unsafe package entries as normal package names, making `package init` fail instead of reporting the project ready when unsafe package names are already present, making `package add` fail before changing files when unsafe package names are already present, making `package remove` fail instead of reporting success when unsafe package names remain during lockfile refresh, allowing `package remove` to remove an exact unsafe package entry so learners can repair a bad `claro.project`, making `package doctor` reject stale lockfile checksums for listed packages, and making `package doctor` reject lockfile package entries that are not listed in `claro.project`. Current function validation covers correct checked calls, wrong-type arguments, missing checked-argument diagnostics, missing unchecked-argument diagnostics for simple functions in both modern `DO greet` and empty compatibility `CALL greet WITH` forms, extra-argument diagnostics, and beginner-facing unknown-function diagnostics for mistyped modern `DO` and compatibility `CALL ... WITH` calls; current object-method parameter validation covers correct modern `DO object.method ...` and compatibility `CALL object.method WITH ...` checked calls, wrong-type diagnostics for both call forms, checked methods after another method in the same class, missing and extra checked-argument diagnostics including the extra-argument case for a checked method after another method in the same class, missing unchecked-argument diagnostics for simple object methods, missing-object guidance when `DO player.method ...` or compatibility `CALL object.method WITH ...` appears before `NEW` even if the method name is also wrong, and class-specific unknown-method diagnostics for both modern and compatibility calls when a learner calls a method the class does not declare; current object-field validation covers direct NUMBER/TEXT/YESNO field-assignment positives, direct object-field `CHECK TYPE` metadata positives for NUMBER/TEXT/YESNO fields, negative NUMBER/TEXT/YESNO field metadata mismatches, NUMBER/TEXT/YESNO-expectation unknown-field `CHECK TYPE` diagnostics, direct wrong-type field assignment diagnostics, field collection when a `HAS` field appears after a simple method, direct unknown-field diagnostics for NUMBER/TEXT/YESNO values, a beginner-facing fallback for unknown fields assigned from expressions whose type is not inferable yet, and missing-object diagnostics for both `SET object.field value` and `CHECK TYPE object.field IS TYPE` before `NEW`. +3. Expand validation around typecheck diagnostics and package/networking safety. Current Forgejo/Gitea CI runs the documented release gates (`claro validate`, typecheck diagnostics validation, version convention validation, package security validation, and CI workflow coverage validation). The workflow coverage validator checks executable `run` steps rather than comments. Current package safety validation covers safe project creation, rejecting unsafe project names during `claro new`, safe manifest/lockfile creation, rejecting unsafe package names during `package add`, detecting unsafe package names already present in `claro.project` during `package doctor`, refusing to write lockfile data for unsafe package names during `package lock`, making `package list` fail instead of displaying unsafe package entries as normal package names, making `package init` fail instead of reporting the project ready when unsafe package names are already present, making `package add` fail before changing files when unsafe package names are already present, making `package remove` fail instead of reporting success when unsafe package names remain during lockfile refresh, allowing `package remove` to remove an exact unsafe package entry so learners can repair a bad `claro.project`, making `package doctor` reject stale lockfile checksums for listed packages, and making `package doctor` reject lockfile package entries that are not listed in `claro.project`. Current function validation covers correct checked calls, wrong-type arguments, missing checked-argument diagnostics, missing unchecked-argument diagnostics for simple functions in both modern `DO greet` and empty compatibility `CALL greet WITH` forms, extra-argument diagnostics, and beginner-facing unknown-function diagnostics for mistyped modern `DO` and compatibility `CALL ... WITH` calls; current object-method parameter validation covers correct modern `DO object.method ...` and compatibility `CALL object.method WITH ...` checked calls, wrong-type diagnostics for both call forms, checked methods after another method in the same class, missing and extra checked-argument diagnostics including the extra-argument case for a checked method after another method in the same class, missing unchecked-argument diagnostics for simple object methods, missing-object guidance when `DO player.method ...` or compatibility `CALL object.method WITH ...` appears before `NEW` even if the method name is also wrong, and class-specific unknown-method diagnostics for both modern and compatibility calls when a learner calls a method the class does not declare; current object-field validation covers direct NUMBER/TEXT/YESNO field-assignment positives, direct object-field `CHECK TYPE` metadata positives for NUMBER/TEXT/YESNO fields, negative NUMBER/TEXT/YESNO field metadata mismatches, NUMBER/TEXT/YESNO-expectation unknown-field `CHECK TYPE` diagnostics, direct wrong-type field assignment diagnostics, field collection when a `HAS` field appears after a simple method, direct unknown-field diagnostics for NUMBER/TEXT/YESNO values, a beginner-facing fallback for unknown fields assigned from expressions whose type is not inferable yet, and missing-object diagnostics for both `SET object.field value` and `CHECK TYPE object.field IS TYPE` before `NEW`. 3a. Keep package diagnostics actionable: invalid package manifest format fields now name the file and explain how to repair it, separately from an absent manifest. Keep package validation honest by checking that the project manifest has one non-empty project name, that the project and each listed package manifest declare the exact supported manifest version, package version, local source, and expected package name, and that lockfiles declare exactly one current release version. 3b. Keep package security validation in Forgejo/Gitea CI so unsafe names, exact manifest-version/name/version/checksum mismatches, missing manifests, duplicate project manifest versions and packages, duplicate lock packages, duplicate package manifest names, versions, and checksums, and lockfile errors remain release blockers. 4. Add small examples for each foundation feature before adding bigger syntax. The object-field foundation now includes positive and negative validation for field expressions such as `SET player.score player.name`, `SET player.score player.score + 1`, `SET player.name player.score + 1`, explicitly typed field assignments, simple and chained aliases such as `SET alias player`, `SET backup alias`, followed by `SET backup.score ...` or `CHECK TYPE backup.score IS ...`, chained aliases in object-method calls through both `DO` and compatibility `CALL ... WITH`, and arithmetic/text expression mismatches; typed containers also accept a map with nested type metadata when it is added to a `LIST OF MAP`; broader alias/control-flow checking remains planned. diff --git a/tests/test_validate_ci_workflow.py b/tests/test_validate_ci_workflow.py new file mode 100644 index 0000000..4cde233 --- /dev/null +++ b/tests/test_validate_ci_workflow.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Regression tests for the CI workflow release-gate validator.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "validate_ci_workflow", ROOT / "tools" / "validate_ci_workflow.py" +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class ValidateCiWorkflowTests(unittest.TestCase): + def test_accepts_release_gate_commands_in_run_steps(self): + workflow = """ + steps: + - name: Build + run: make + - name: Release gates + run: ./claro validate + - name: Type diagnostics + run: python3 tools/validate_typecheck_diagnostics.py + - name: Version + run: python3 tools/validate_version_convention.py + - name: Packages + run: python3 tools/validate_package_security.py + """ + self.assertEqual(MODULE.missing_release_gates(workflow), []) + + def test_rejects_commands_that_only_appear_in_comments(self): + workflow = """ + steps: + # run: ./claro validate + # python3 tools/validate_typecheck_diagnostics.py + - name: Build + run: make + """ + self.assertEqual( + MODULE.missing_release_gates(workflow), MODULE.REQUIRED_COMMANDS + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate_ci_workflow.py b/tools/validate_ci_workflow.py index a89ed76..3ff732c 100644 --- a/tools/validate_ci_workflow.py +++ b/tools/validate_ci_workflow.py @@ -14,13 +14,43 @@ REQUIRED_COMMANDS = [ ] +def _run_step_text(workflow_text: str) -> str: + """Return commands from YAML-like ``run`` steps, excluding comments/names.""" + commands = [] + in_run = False + run_indent = -1 + for raw_line in workflow_text.splitlines(): + stripped = raw_line.lstrip() + if not stripped or stripped.startswith("#"): + continue + indent = len(raw_line) - len(stripped) + if stripped.startswith("run:"): + in_run = True + run_indent = indent + commands.append(stripped[4:].strip()) + continue + if in_run: + if indent <= run_indent and stripped.startswith("-"): + in_run = False + elif indent <= run_indent and ":" in stripped: + in_run = False + else: + commands.append(stripped) + return "\n".join(commands) + + +def missing_release_gates(workflow_text: str): + run_text = _run_step_text(workflow_text) + return [command for command in REQUIRED_COMMANDS if command not in run_text] + + def main() -> int: if not WORKFLOW.exists(): print(f"Missing CI workflow: {WORKFLOW.relative_to(ROOT)}") return 1 text = WORKFLOW.read_text(encoding="utf-8") - missing = [command for command in REQUIRED_COMMANDS if command not in text] + missing = missing_release_gates(text) if missing: print("CI workflow is missing current release validation commands:") for command in missing: