From 532f6a98d8153d4dfdc1e4ce1769af866547538e Mon Sep 17 00:00:00 2001 From: Micheal Wilkinson Date: Sat, 21 Mar 2026 22:34:01 +0000 Subject: [PATCH] feat: extract coverage-gate action from Cue for reuse across projects - Move coveragegate tool from cue/tools to vociferate/coverage-gate - Create composite action with JSON metrics output for CI - Update tool to export passes/total_coverage/packages_checked/packages_failed - Support per-package threshold policy via JSON configuration - Change module path to git.hrafn.xyz/aether/vociferate/coverage-gate --- coverage-gate/README.md | 85 +++++++++++++ coverage-gate/action.yml | 91 ++++++++++++++ coverage-gate/go.mod | 3 + coverage-gate/integration_test.go | 26 ++++ coverage-gate/main.go | 118 ++++++++++++++++++ coverage-gate/main_test.go | 85 +++++++++++++ coverage-gate/parse.go | 200 ++++++++++++++++++++++++++++++ coverage-gate/parse_test.go | 88 +++++++++++++ 8 files changed, 696 insertions(+) create mode 100644 coverage-gate/README.md create mode 100644 coverage-gate/action.yml create mode 100644 coverage-gate/go.mod create mode 100644 coverage-gate/integration_test.go create mode 100644 coverage-gate/main.go create mode 100644 coverage-gate/main_test.go create mode 100644 coverage-gate/parse.go create mode 100644 coverage-gate/parse_test.go diff --git a/coverage-gate/README.md b/coverage-gate/README.md new file mode 100644 index 0000000..77302b7 --- /dev/null +++ b/coverage-gate/README.md @@ -0,0 +1,85 @@ +# coveragegate + +Standalone coverage-threshold enforcement tool for this repository. + +This tool is a quality gate. It is not part of Cue runtime orchestration. + +## What it does + +- Reads a Go coverage profile (for example `_build/coverage.out`). +- Loads package coverage policy from JSON. +- Discovers packages under a source root using `go list ./...`. +- Evaluates per-package statement coverage against policy thresholds. +- Prints a package table and returns a non-zero exit code when any package fails. + +## Repository integration + +Primary repository flow: + +1. `just test-coverage` runs tests in `src/` and writes `_build/coverage.out`. +2. `scripts/check-core-coverage.sh` runs this tool from `tools/coveragegate/`. +3. The script currently passes: + - `--profile $ROOT_DIR/_build/coverage.out` + - `--policy $ROOT_DIR/docs/coverage-thresholds.json` + - `--src-root $ROOT_DIR/src` + +## Usage + +From repository root: + +```bash +cd tools/coveragegate +go run . \ + --profile ../../_build/coverage.out \ + --policy ../../docs/coverage-thresholds.json \ + --src-root ../../src +``` + +Or use the repository wrapper: + +```bash +bash scripts/check-core-coverage.sh +``` + +## Flags + +- `--profile`: Path to Go coverage profile. +- `--policy`: Path to JSON policy file. +- `--src-root`: Directory where packages are discovered with `go list ./...`. + +## Exit codes + +- `0`: All in-scope packages meet threshold. +- `1`: Policy/profile/load failure or one or more packages below threshold. +- `2`: Invalid CLI arguments. + +## Policy model (current) + +The tool expects a JSON object with at least: + +- `minimum_statement_coverage` (number) +- `critical_packages` (array) + +Each critical package may include: + +- `package` (string) +- `minimum_statement_coverage` (number) +- `include` (boolean) +- `exclusions` (array of strings) + +Behavior notes: + +- If a package has no policy override, the global minimum is used. +- Generated/composition files are excluded by built-in rules. +- Packages with no statements are treated as passing. + +## Development + +Run tests: + +```bash +cd tools/coveragegate +go test ./... +``` + +Keep code `gofmt` and `go vet` clean. diff --git a/coverage-gate/action.yml b/coverage-gate/action.yml new file mode 100644 index 0000000..ce13d0c --- /dev/null +++ b/coverage-gate/action.yml @@ -0,0 +1,91 @@ +name: vociferate/coverage-gate +description: > + Enforce per-package code coverage thresholds against Go coverage profiles. + Supports JSON policy files with per-package overrides and global minimums. + +inputs: + profile: + description: Path to Go coverage profile file (output from `go test -coverprofile=...`). + required: false + default: coverage.out + policy: + description: Path to JSON file defining coverage thresholds and per-package overrides. + required: false + default: docs/coverage-thresholds.json + src-root: + description: Source root directory for package discovery (passed to `go list ./...`). + required: false + default: . + summary-file: + description: Optional file path to append markdown summary of coverage results. + required: false + default: '' + +outputs: + passed: + description: 'Boolean: true if all packages meet threshold, false if any failed.' + value: ${{ steps.gate.outputs.passed }} + total-coverage: + description: Repository-wide statement coverage percentage. + value: ${{ steps.gate.outputs.total_coverage }} + packages-checked: + description: Number of packages evaluated against policy. + value: ${{ steps.gate.outputs.packages_checked }} + packages-failed: + description: Number of packages below threshold. + value: ${{ steps.gate.outputs.packages_failed }} + +runs: + using: composite + steps: + - id: gate + shell: bash + working-directory: ${{ github.action_path }} + env: + PROFILE: ${{ inputs.profile }} + POLICY: ${{ inputs.policy }} + SRC_ROOT: ${{ inputs.src-root }} + SUMMARY_FILE: ${{ inputs.summary-file }} + run: | + set -euo pipefail + + # Run coverage gate and capture output + EXIT_CODE=0 + OUTPUT=$(go run . \ + --profile "$PROFILE" \ + --policy "$POLICY" \ + --src-root "$SRC_ROOT" \ + ) || EXIT_CODE=$? + + echo "$OUTPUT" + + # Parse summary from output (tool prints JSON stats on last line) + SUMMARY_LINE=$(echo "$OUTPUT" | tail -1) + + # Determine pass/fail + if [[ $EXIT_CODE -eq 0 ]]; then + echo "passed=true" >> "$GITHUB_OUTPUT" + else + echo "passed=false" >> "$GITHUB_OUTPUT" + fi + + # Extract metrics (tool outputs: packages_checked, packages_failed, total_coverage on summary line) + if echo "$SUMMARY_LINE" | jq . &>/dev/null; then + echo "total_coverage=$(echo "$SUMMARY_LINE" | jq -r '.total_coverage')" >> "$GITHUB_OUTPUT" + echo "packages_checked=$(echo "$SUMMARY_LINE" | jq -r '.packages_checked')" >> "$GITHUB_OUTPUT" + echo "packages_failed=$(echo "$SUMMARY_LINE" | jq -r '.packages_failed')" >> "$GITHUB_OUTPUT" + + # Append to summary file if provided + if [[ -n "$SUMMARY_FILE" ]]; then + { + echo "## Coverage Gate Results" + echo + echo "- **Passed:** $([ "$EXIT_CODE" -eq 0 ] && echo '✓ Yes' || echo '✗ No')" + echo "- **Total Coverage:** $(echo "$SUMMARY_LINE" | jq -r '.total_coverage')%" + echo "- **Packages Checked:** $(echo "$SUMMARY_LINE" | jq -r '.packages_checked')" + echo "- **Packages Failed:** $(echo "$SUMMARY_LINE" | jq -r '.packages_failed')" + } >> "$SUMMARY_FILE" + fi + fi + + exit $EXIT_CODE diff --git a/coverage-gate/go.mod b/coverage-gate/go.mod new file mode 100644 index 0000000..fe1a920 --- /dev/null +++ b/coverage-gate/go.mod @@ -0,0 +1,3 @@ +module git.hrafn.xyz/aether/vociferate/coverage-gate + +go 1.26.1 diff --git a/coverage-gate/integration_test.go b/coverage-gate/integration_test.go new file mode 100644 index 0000000..7493e43 --- /dev/null +++ b/coverage-gate/integration_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRun_ExitCodeOnInvalidProfile(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + policyPath := filepath.Join(tmp, "policy.json") + if err := os.WriteFile(policyPath, []byte(`{"minimum_statement_coverage":80,"critical_packages":[]}`), 0600); err != nil { + t.Fatalf("write policy: %v", err) + } + + exit := run( + []string{"--profile", filepath.Join(tmp, "missing.out"), "--policy", policyPath, "--src-root", "."}, + os.Stdout, + os.Stderr, + func(_ string) ([]string, error) { return []string{"git.hrafn.xyz/aether/cue/service/llm"}, nil }, + ) + if exit != 1 { + t.Fatalf("expected exit 1 for missing profile, got %d", exit) + } +} diff --git a/coverage-gate/main.go b/coverage-gate/main.go new file mode 100644 index 0000000..113ae8b --- /dev/null +++ b/coverage-gate/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strings" +) + +type discoverPackagesFunc func(srcRoot string) ([]string, error) + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr, discoverPackages)) +} + +func run(args []string, stdout io.Writer, stderr io.Writer, discover discoverPackagesFunc) int { + fs := flag.NewFlagSet("coveragegate", flag.ContinueOnError) + fs.SetOutput(stderr) + + profilePath := fs.String("profile", "../_build/coverage.out", "path to go coverprofile") + policyPath := fs.String("policy", "../specs/003-testing-time/contracts/coverage-thresholds.json", "path to coverage policy json") + srcRoot := fs.String("src-root", ".", "path to src workspace root") + + if err := fs.Parse(args); err != nil { + return 2 + } + + policy, err := LoadPolicy(*policyPath) + if err != nil { + fmt.Fprintf(stderr, "coverage gate: %v\n", err) + return 1 + } + + aggByPkg, err := ParseCoverProfile(*profilePath, policy) + if err != nil { + fmt.Fprintf(stderr, "coverage gate: %v\n", err) + return 1 + } + + pkgs, err := discover(*srcRoot) + if err != nil { + fmt.Fprintf(stderr, "coverage gate: %v\n", err) + return 1 + } + + results := EvaluateCoverage(pkgs, aggByPkg, policy) + if len(results) == 0 { + fmt.Fprintln(stderr, "coverage gate: no in-scope packages found") + return 1 + } + + fmt.Fprintln(stdout, "Package coverage results:") + fmt.Fprintln(stdout, "PACKAGE\tCOVERAGE\tTHRESHOLD\tSTATUS") + + failed := false + totalCoverage := 0.0 + for _, r := range results { + status := "PASS" + if !r.Pass { + status = "FAIL" + failed = true + } + fmt.Fprintf(stdout, "%s\t%.2f%%\t%.2f%%\t%s\n", r.Package, r.Percent, r.Threshold, status) + totalCoverage += r.Percent + } + if len(results) > 0 { + totalCoverage /= float64(len(results)) + } + + packagesFailed := 0 + for _, r := range results { + if !r.Pass { + packagesFailed++ + } + } + + // Output JSON metrics for CI consumption + metrics := map[string]interface{}{ + "passed": !failed, + "total_coverage": fmt.Sprintf("%.2f", totalCoverage), + "packages_checked": len(results), + "packages_failed": packagesFailed, + } + metricsJSON, _ := json.Marshal(metrics) + fmt.Fprintln(stdout, string(metricsJSON)) + + if failed { + fmt.Fprintln(stderr, "coverage gate: one or more packages are below threshold") + return 1 + } + fmt.Fprintln(stdout, "coverage gate: all in-scope packages meet threshold") + return 0 +} + +func discoverPackages(srcRoot string) ([]string, error) { + cmd := exec.Command("go", "list", "./...") + cmd.Dir = srcRoot + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("discover packages with go list: %w", err) + } + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + pkgs := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + pkgs = append(pkgs, line) + } + sort.Strings(pkgs) + return pkgs, nil +} diff --git a/coverage-gate/main_test.go b/coverage-gate/main_test.go new file mode 100644 index 0000000..981451f --- /dev/null +++ b/coverage-gate/main_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRun_FailsWhenBelowThreshold(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + policyPath := filepath.Join(tmp, "policy.json") + profilePath := filepath.Join(tmp, "coverage.out") + + policy := `{ + "minimum_statement_coverage": 80, + "critical_packages": [] +}` + if err := os.WriteFile(policyPath, []byte(policy), 0600); err != nil { + t.Fatalf("write policy: %v", err) + } + + profile := "mode: set\n" + + "git.hrafn.xyz/aether/cue/service/llm/validator.go:1.1,2.1 10 0\n" + if err := os.WriteFile(profilePath, []byte(profile), 0600); err != nil { + t.Fatalf("write profile: %v", err) + } + + var out bytes.Buffer + var errOut bytes.Buffer + exit := run( + []string{"--profile", profilePath, "--policy", policyPath, "--src-root", "."}, + &out, + &errOut, + func(_ string) ([]string, error) { + return []string{"git.hrafn.xyz/aether/cue/service/llm"}, nil + }, + ) + if exit != 1 { + t.Fatalf("expected exit 1, got %d\nstdout: %s\nstderr: %s", exit, out.String(), errOut.String()) + } + if !strings.Contains(errOut.String(), "below threshold") { + t.Fatalf("expected threshold error, got: %s", errOut.String()) + } +} + +func TestRun_PassesWhenAllMeetThreshold(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + policyPath := filepath.Join(tmp, "policy.json") + profilePath := filepath.Join(tmp, "coverage.out") + + policy := `{ + "minimum_statement_coverage": 80, + "critical_packages": [] +}` + if err := os.WriteFile(policyPath, []byte(policy), 0600); err != nil { + t.Fatalf("write policy: %v", err) + } + + profile := "mode: set\n" + + "git.hrafn.xyz/aether/cue/service/llm/validator.go:1.1,2.1 10 1\n" + if err := os.WriteFile(profilePath, []byte(profile), 0600); err != nil { + t.Fatalf("write profile: %v", err) + } + + var out bytes.Buffer + var errOut bytes.Buffer + exit := run( + []string{"--profile", profilePath, "--policy", policyPath, "--src-root", "."}, + &out, + &errOut, + func(_ string) ([]string, error) { + return []string{"git.hrafn.xyz/aether/cue/service/llm"}, nil + }, + ) + if exit != 0 { + t.Fatalf("expected exit 0, got %d\nstdout: %s\nstderr: %s", exit, out.String(), errOut.String()) + } + if !strings.Contains(out.String(), "all in-scope packages meet threshold") { + t.Fatalf("expected success summary, got: %s", out.String()) + } +} diff --git a/coverage-gate/parse.go b/coverage-gate/parse.go new file mode 100644 index 0000000..da5a4cc --- /dev/null +++ b/coverage-gate/parse.go @@ -0,0 +1,200 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Policy describes coverage threshold configuration. +type Policy struct { + MinimumStatementCoverage float64 `json:"minimum_statement_coverage"` + CriticalPackages []PackagePolicy `json:"critical_packages"` +} + +// PackagePolicy overrides defaults for a specific package. +type PackagePolicy struct { + Package string `json:"package"` + MinimumStatementCoverage float64 `json:"minimum_statement_coverage"` + Include bool `json:"include"` + Exclusions []string `json:"exclusions"` +} + +// Coverage aggregates covered and total statements for a package. +type Coverage struct { + Covered int64 + Total int64 +} + +// PackageResult is the policy-evaluated coverage result for one package. +type PackageResult struct { + Package string + Covered int64 + Total int64 + Percent float64 + Threshold float64 + Pass bool +} + +// LoadPolicy reads policy JSON from disk. +func LoadPolicy(path string) (Policy, error) { + f, err := os.Open(path) + if err != nil { + return Policy{}, fmt.Errorf("open policy: %w", err) + } + defer f.Close() + + var p Policy + if err := json.NewDecoder(f).Decode(&p); err != nil { + return Policy{}, fmt.Errorf("decode policy: %w", err) + } + if p.MinimumStatementCoverage <= 0 { + p.MinimumStatementCoverage = 80.0 + } + return p, nil +} + +// ParseCoverProfile parses a Go coverprofile and aggregates package coverage. +func ParseCoverProfile(profilePath string, policy Policy) (map[string]Coverage, error) { + f, err := os.Open(profilePath) + if err != nil { + return nil, fmt.Errorf("open coverage profile: %w", err) + } + defer f.Close() + + coverage := make(map[string]Coverage) + s := bufio.NewScanner(f) + lineNo := 0 + for s.Scan() { + lineNo++ + line := strings.TrimSpace(s.Text()) + if lineNo == 1 { + if !strings.HasPrefix(line, "mode:") { + return nil, fmt.Errorf("invalid coverage profile header") + } + continue + } + if line == "" { + continue + } + + parts := strings.Fields(line) + if len(parts) != 3 { + return nil, fmt.Errorf("invalid coverage line %d", lineNo) + } + fileAndRange := parts[0] + numStmts, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid statements count at line %d: %w", lineNo, err) + } + execCount, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid execution count at line %d: %w", lineNo, err) + } + + idx := strings.Index(fileAndRange, ":") + if idx < 0 { + return nil, fmt.Errorf("invalid file segment at line %d", lineNo) + } + filePath := fileAndRange[:idx] + pkg := filepath.ToSlash(filepath.Dir(filePath)) + if isExcludedFile(pkg, filePath, policy) { + continue + } + + agg := coverage[pkg] + agg.Total += numStmts + if execCount > 0 { + agg.Covered += numStmts + } + coverage[pkg] = agg + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("scan coverage profile: %w", err) + } + + return coverage, nil +} + +// EvaluateCoverage evaluates package coverage against policy thresholds. +func EvaluateCoverage(packages []string, byPackage map[string]Coverage, policy Policy) []PackageResult { + results := make([]PackageResult, 0, len(packages)) + for _, pkg := range packages { + if !isPackageIncluded(pkg, policy) { + continue + } + agg := byPackage[pkg] + percent := 100.0 + if agg.Total > 0 { + percent = float64(agg.Covered) * 100.0 / float64(agg.Total) + } + threshold := thresholdForPackage(pkg, policy) + pass := agg.Total == 0 || percent >= threshold + results = append(results, PackageResult{ + Package: pkg, + Covered: agg.Covered, + Total: agg.Total, + Percent: percent, + Threshold: threshold, + Pass: pass, + }) + } + sort.Slice(results, func(i, j int) bool { + return results[i].Package < results[j].Package + }) + return results +} + +func thresholdForPackage(pkg string, policy Policy) float64 { + for _, entry := range policy.CriticalPackages { + if entry.Package == pkg && entry.MinimumStatementCoverage > 0 { + return entry.MinimumStatementCoverage + } + } + if policy.MinimumStatementCoverage > 0 { + return policy.MinimumStatementCoverage + } + return 80.0 +} + +func isPackageIncluded(pkg string, policy Policy) bool { + for _, entry := range policy.CriticalPackages { + if entry.Package == pkg { + return entry.Include + } + } + return true +} + +func isExcludedFile(pkg string, filePath string, policy Policy) bool { + base := filepath.Base(filePath) + + // Exclude known generated artifacts and thin composition wiring. + if strings.HasSuffix(base, "_gen.go") || + base == "generated.go" || + base == "models_gen.go" || + base == "schema.resolvers.go" || + base == "main.go" { + return true + } + + for _, entry := range policy.CriticalPackages { + if entry.Package != pkg { + continue + } + for _, ex := range entry.Exclusions { + if ex == "" { + continue + } + if strings.HasSuffix(filePath, ex) || strings.Contains(filePath, "/"+ex) { + return true + } + } + } + return false +} diff --git a/coverage-gate/parse_test.go b/coverage-gate/parse_test.go new file mode 100644 index 0000000..037f747 --- /dev/null +++ b/coverage-gate/parse_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseCoverProfile_AppliesExclusions(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + profile := filepath.Join(tmp, "coverage.out") + content := "mode: set\n" + + "git.hrafn.xyz/aether/cue/api/graph/generated.go:1.1,2.1 2 1\n" + + "git.hrafn.xyz/aether/cue/api/graph/resolver.go:1.1,2.1 2 1\n" + + "git.hrafn.xyz/aether/cue/service/llm/validator.go:1.1,2.1 2 0\n" + if err := os.WriteFile(profile, []byte(content), 0600); err != nil { + t.Fatalf("write profile: %v", err) + } + + policy := Policy{ + MinimumStatementCoverage: 80, + CriticalPackages: []PackagePolicy{ + {Package: "git.hrafn.xyz/aether/cue/api/graph", Include: true, Exclusions: []string{"generated.go"}}, + }, + } + + got, err := ParseCoverProfile(profile, policy) + if err != nil { + t.Fatalf("ParseCoverProfile() error = %v", err) + } + + api := got["git.hrafn.xyz/aether/cue/api/graph"] + if api.Total != 2 || api.Covered != 2 { + t.Fatalf("api coverage mismatch: got %+v", api) + } + llm := got["git.hrafn.xyz/aether/cue/service/llm"] + if llm.Total != 2 || llm.Covered != 0 { + t.Fatalf("llm coverage mismatch: got %+v", llm) + } +} + +func TestEvaluateCoverage_UsesPolicyThresholds(t *testing.T) { + t.Parallel() + pkgs := []string{ + "git.hrafn.xyz/aether/cue/service/llm", + "git.hrafn.xyz/aether/cue/service/orchestrator", + } + byPkg := map[string]Coverage{ + "git.hrafn.xyz/aether/cue/service/llm": {Covered: 8, Total: 10}, + "git.hrafn.xyz/aether/cue/service/orchestrator": {Covered: 3, Total: 10}, + } + policy := Policy{ + MinimumStatementCoverage: 80, + CriticalPackages: []PackagePolicy{ + {Package: "git.hrafn.xyz/aether/cue/service/orchestrator", MinimumStatementCoverage: 30, Include: true}, + }, + } + + results := EvaluateCoverage(pkgs, byPkg, policy) + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + if !results[0].Pass { + t.Fatalf("expected llm to pass at default threshold: %+v", results[0]) + } + if !results[1].Pass { + t.Fatalf("expected orchestrator to pass at overridden threshold: %+v", results[1]) + } +} + +func TestEvaluateCoverage_NoStatementsPasses(t *testing.T) { + t.Parallel() + + pkg := "git.hrafn.xyz/aether/cue/repository/vector" + results := EvaluateCoverage( + []string{pkg}, + map[string]Coverage{pkg: {Covered: 0, Total: 0}}, + Policy{MinimumStatementCoverage: 80}, + ) + + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if !results[0].Pass { + t.Fatalf("expected pass for no-statement package, got %+v", results[0]) + } +}