diff --git a/README.md b/README.md index d87255f..6c512d0 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,9 @@ canopy search refs ParseConfig . # Check code quality (CI gate) canopy analyze check --max-cyclomatic 30 +# Inspect parser gaps and fail on incomplete or unknown receipts +canopy index coverage . --strict + # Full executive report canopy analyze report --format markdown @@ -37,7 +40,9 @@ canopy mcp --root . ## Current Main -Current main uses gotreesitter `v0.45.0` and keeps large-repository indexing bounded by default: the CLI uses a 1 GiB Go soft memory limit when the caller has not supplied one, while index builds use at most two concurrent parse workers and a garbage collection cadence of 32 parsed files. Index walks also prune ignored directories before descent and skip unsupported or tagless grammars before parsing. +Current main uses gotreesitter `v0.49.0` and keeps large-repository indexing bounded by default: the CLI uses a 1 GiB Go soft memory limit when the caller has not supplied one, while index builds use at most two concurrent parse workers and a garbage collection cadence of 32 parsed files. Index walks also prune ignored directories before descent and skip unsupported or tagless grammars before parsing. + +Index schema `0.3.0` stores parser-health receipts for each file. Receipts preserve top-most `ERROR` and `MISSING` regions, parser stop reasons, recovered-region decisions, and generated-file fast paths. A `clean` receipt means no actionable parser gap was detected; it is not a proof that a grammar or tags query models every construct. Diff-aware checks and reviews use a valid repository index when one exists. Without one, they build and report a changed-only snapshot. @@ -64,6 +69,7 @@ Call graph roots can be narrowed with `--file` or `path/to/file.go:Name` when mu | `canopy index stats` | Codebase metrics: symbol counts, language breakdown | | `canopy index diff` | Compare structural changes between two snapshots | | `canopy index errors` | Show parse errors from indexing | +| `canopy index coverage` | Report parser gaps and missing parse receipts. `--strict` provides a CI gate | | `canopy index validate` | Validate index integrity | | `canopy index export` | Export index to portable `.canopyindex` file for federation | | `canopy index import` | Load and summarize exported indexes | @@ -76,7 +82,7 @@ Call graph roots can be narrowed with `--file` or `path/to/file.go:Name` when mu | `canopy search refs` | Find references by symbol name or regex | | `canopy search query` | Raw tree-sitter S-expression queries | | `canopy search scope` | Resolve symbols in scope at file + line | -| `canopy search context` | Pack focused context for agent token budgets. `--concept` for concept-aware packing | +| `canopy search context` | Pack focused context for agent token budgets. Bundle mode reports parser health; `--concept` enables concept-aware packing | | `canopy search symbols` | Search symbols by pattern | | `canopy search imports` | Analyze import patterns | @@ -109,8 +115,8 @@ Call graph roots can be narrowed with `--file` or `path/to/file.go:Name` when mu | `canopy analyze licenses` | Dependency license detection with SPDX matching and deny rules | | `canopy analyze similarity` | Find similar functions between codebases | | `canopy analyze duplication` | Detect code duplication | -| `canopy analyze report` | Executive summary: complexity, architecture, security, dead code, hotspots. `--by-team` for CODEOWNERS breakdown | -| `canopy analyze review` | Aggregated PR review: complexity delta, boundary violations, new capabilities, blast radius | +| `canopy analyze report` | Confidence-aware summary: parser health, complexity, architecture, security, dead code, hotspots. `--by-team` for CODEOWNERS breakdown | +| `canopy analyze review` | Confidence-aware PR review: parser health, complexity delta, boundary violations, new capabilities, blast radius | | `canopy analyze trends` | Track quality metrics over time (`record` / `show`) | ### Transform — Code transformations and output generation @@ -257,8 +263,9 @@ canopy mcp --root /path/to/repo --allow-writes # enable refactoring tools | `gts_callgraph` | Call graph traversal | | `gts_dead` | Dead code detection | | `gts_impact` | Blast radius computation | -| `gts_context` | Token-budgeted context packing | +| `gts_context` | Token-budgeted context packing with parser health on task-conditioned bundles | | `gts_grep` | Structural selector search | +| `gts_coverage` | Parser gaps, stopped parses, recovery receipts, and unknown coverage | ## Selector Syntax diff --git a/cmd/canopy/context.go b/cmd/canopy/context.go index 0f2c643..68f7eb1 100644 --- a/cmd/canopy/context.go +++ b/cmd/canopy/context.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "m31labs.dev/canopy/internal/contextpack" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/contextbundle" "m31labs.dev/canopy/pkg/model" "m31labs.dev/canopy/pkg/xref" @@ -77,6 +78,10 @@ func newContextCmd() *cobra.Command { if err != nil { return err } + parserHealth, err := indexcoverage.BuildHealthForPaths(idx, contextManifestPaths(result.Manifest), 5) + if err != nil { + return err + } if bundleManifestPath != "" { if err := writeJSONFile(bundleManifestPath, result.Manifest); err != nil { @@ -90,9 +95,14 @@ func newContextCmd() *cobra.Command { } if jsonOutput { return emitJSON(struct { - Receipt contextbundle.Receipt `json:"receipt"` - Manifest contextbundle.Manifest `json:"manifest"` - }{Receipt: result.Receipt, Manifest: result.Manifest}) + Receipt contextbundle.Receipt `json:"receipt"` + Manifest contextbundle.Manifest `json:"manifest"` + ParserHealth indexcoverage.Health `json:"parser_health"` + }{Receipt: result.Receipt, Manifest: result.Manifest, ParserHealth: parserHealth}) + } + if parserHealth.Status != model.ParseCoverageClean && parserHealth.Status != model.ParseCoverageGenerated { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: selected context parser health is %s (%d untrusted files)\n", + parserHealth.Status, parserHealth.Summary.Untrusted) } fmt.Print(string(result.Content)) return nil @@ -201,6 +211,14 @@ func newContextCmd() *cobra.Command { return cmd } +func contextManifestPaths(manifest contextbundle.Manifest) []string { + paths := make([]string, 0, len(manifest.Items)) + for _, item := range manifest.Items { + paths = append(paths, item.Path) + } + return paths +} + // conceptReport holds the result of concept-aware context packing. type conceptReport struct { Concept string `json:"concept"` diff --git a/cmd/canopy/group_index.go b/cmd/canopy/group_index.go index 1631852..65078f4 100644 --- a/cmd/canopy/group_index.go +++ b/cmd/canopy/group_index.go @@ -14,6 +14,7 @@ func newIndexGroup() *cobra.Command { newStatsCmd(), newDiffCmd(), newErrorsCmd(), + newIndexCoverageCmd(), newValidateCmd(), newExportCmd(), newImportCmd(), diff --git a/cmd/canopy/index.go b/cmd/canopy/index.go index f0b5d75..bb46bf4 100644 --- a/cmd/canopy/index.go +++ b/cmd/canopy/index.go @@ -166,7 +166,9 @@ func loadBaselineIndex(outPath string) (*model.Index, bool, error) { if strings.TrimSpace(outPath) == "" { return nil, false, nil } - cached, err := index.Load(outPath) + // A build can migrate an older cache by treating it as a comparison + // baseline while the builder reparses entries from the current schema. + cached, err := index.LoadLenient(outPath) switch { case err == nil: return cached, true, nil @@ -201,7 +203,9 @@ func compareBaseline(previous, idx *model.Index, hasBaseline bool) (structdiff.R changed := true if hasBaseline { report = structdiff.Compare(previous, idx) - changed = report.Stats.ChangedFiles > 0 || !parseErrorsEqual(previous.Errors, idx.Errors) + changed = report.Stats.ChangedFiles > 0 || + !parseErrorsEqual(previous.Errors, idx.Errors) || + !parseCoverageEqual(previous.Files, idx.Files) } return report, changed } @@ -239,7 +243,9 @@ func runIndexWatch(ctx context.Context, target string, builder *index.Builder, c } watchReport := structdiff.Compare(current, next) - watchChanged := watchReport.Stats.ChangedFiles > 0 || !parseErrorsEqual(current.Errors, next.Errors) + watchChanged := watchReport.Stats.ChangedFiles > 0 || + !parseErrorsEqual(current.Errors, next.Errors) || + !parseCoverageEqual(current.Files, next.Files) if !watchChanged { return } diff --git a/cmd/canopy/index_coverage.go b/cmd/canopy/index_coverage.go new file mode 100644 index 0000000..cb082e4 --- /dev/null +++ b/cmd/canopy/index_coverage.go @@ -0,0 +1,149 @@ +package main + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "m31labs.dev/canopy/internal/indexcoverage" + "m31labs.dev/canopy/pkg/model" +) + +func newIndexCoverageCmd() *cobra.Command { + var cachePath string + var noCache bool + var jsonOutput bool + var includeAll bool + var strict bool + var limit int + + cmd := &cobra.Command{ + Use: "coverage [path]", + Short: "Report parser coverage gaps and missing parse receipts", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if limit <= 0 { + return fmt.Errorf("limit must be > 0") + } + target := "." + if len(args) == 1 { + target = args[0] + } + + idx, err := loadOrBuild(cmd, cachePath, target, noCache) + if err != nil { + return err + } + if generator, _ := cmd.Flags().GetString("generator"); generator != "" { + idx = idx.FilterByGenerator(generator) + } + includeGenerated, _ := cmd.Flags().GetBool("include-generated") + report, err := indexcoverage.Build(idx, indexcoverage.Options{ + IncludeClean: includeAll, + IncludeGenerated: includeAll || includeGenerated, + MaxFiles: limit, + }) + if err != nil { + return err + } + + if jsonOutput { + if err := emitJSON(report); err != nil { + return err + } + } else { + printCoverageReport(report) + } + if strict && report.StrictFailure() { + return exitCodeError{code: 2, err: errors.New("index contains incomplete or unknown parser coverage")} + } + return nil + }, + } + + cmd.Flags().StringVar(&cachePath, "cache", "", "load index from cache instead of parsing") + cmd.Flags().BoolVar(&noCache, "no-cache", false, "skip auto-discovery of cached index") + cmd.Flags().BoolVar(&jsonOutput, "json", false, "emit JSON output") + cmd.Flags().BoolVar(&includeAll, "all", false, "include clean and generated-file receipts") + cmd.Flags().BoolVar(&strict, "strict", false, "exit with code 2 for partial, stopped, unknown, or failed parses") + cmd.Flags().IntVar(&limit, "limit", 100, "maximum number of file receipts to emit") + return cmd +} + +func printCoverageReport(report indexcoverage.Report) { + summary := report.Summary + fmt.Printf( + "coverage: files=%d clean=%d partial=%d stopped=%d generated=%d unknown=%d parse_errors=%d gaps=%d root=%s\n", + report.TotalFiles, + summary.Clean, + summary.Partial, + summary.Stopped, + summary.Generated, + summary.Unknown, + summary.ParseErrors, + summary.Gaps, + report.Root, + ) + if summary.Recovered > 0 || summary.IgnoredEOF > 0 || summary.Truncated > 0 { + fmt.Printf( + "receipts: recovered_regions=%d ignored_eof_missing=%d truncated_files=%d\n", + summary.Recovered, + summary.IgnoredEOF, + summary.Truncated, + ) + } + for _, file := range report.Files { + coverage := file.Coverage + fmt.Printf( + " %s status=%s language=%s gaps=%d errors=%d missing=%d", + file.Path, + coverage.Status, + file.Language, + len(coverage.Gaps), + coverage.ErrorNodes, + coverage.MissingNodes, + ) + if coverage.StopReason != "" { + fmt.Printf(" reason=%s", coverage.StopReason) + } + fmt.Println() + for _, gap := range coverage.Gaps { + printCoverageGap(gap) + } + } + if report.DetailsTruncated { + fmt.Println(" ... file receipts truncated; raise --limit to inspect more") + } + if len(report.Errors) > 0 { + fmt.Println("parse errors:") + for _, parseErr := range report.Errors { + fmt.Printf(" %s: %s\n", parseErr.Path, parseErr.Error) + } + } +} + +func printCoverageGap(gap model.ParseGap) { + fmt.Printf( + " %s %d:%d-%d:%d bytes=%d-%d", + gap.Kind, + gap.StartLine, + gap.StartColumn, + gap.EndLine, + gap.EndColumn, + gap.StartByte, + gap.EndByte, + ) + if gap.NodeType != "" { + fmt.Printf(" node=%s", gap.NodeType) + } + fmt.Println() +} + +func runIndexCoverage(args []string) error { + cmd := newIndexCoverageCmd() + cmd.SilenceUsage = true + cmd.SilenceErrors = true + cmd.SetArgs(args) + return cmd.Execute() +} diff --git a/cmd/canopy/index_coverage_test.go b/cmd/canopy/index_coverage_test.go new file mode 100644 index 0000000..df70d53 --- /dev/null +++ b/cmd/canopy/index_coverage_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "m31labs.dev/canopy/internal/indexcoverage" + "m31labs.dev/canopy/pkg/model" +) + +func TestRunIndexCoverageJSON(t *testing.T) { + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "main.go"), []byte("package sample\n\nfunc Work() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile failed: %v", err) + } + + output, runErr := captureIndexCoverageOutput(t, []string{tmpDir, "--no-cache", "--json", "--all"}) + if runErr != nil { + t.Fatalf("runIndexCoverage returned error: %v", runErr) + } + var report indexcoverage.Report + if err := json.Unmarshal(output, &report); err != nil { + t.Fatalf("decode coverage report: %v\n%s", err, output) + } + if report.TotalFiles != 1 || report.Summary.Clean != 1 || len(report.Files) != 1 { + t.Fatalf("unexpected report: %+v", report) + } +} + +func TestRunIndexCoverageStrictRejectsMalformedSource(t *testing.T) { + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "broken.go"), []byte("package sample\n\nfunc Broken( {\n"), 0o644); err != nil { + t.Fatalf("WriteFile failed: %v", err) + } + + _, runErr := captureIndexCoverageOutput(t, []string{tmpDir, "--no-cache", "--strict"}) + if runErr == nil { + t.Fatal("expected strict coverage to fail") + } + assertExitCode(t, runErr, 2) +} + +func TestParseCoverageEqualDetectsReceiptChanges(t *testing.T) { + left := []model.FileSummary{{ + Path: "main.go", + ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}, + }} + right := []model.FileSummary{{ + Path: "main.go", + ParseCoverage: &model.ParseCoverage{Status: model.ParseCoveragePartial}, + }} + if parseCoverageEqual(left, right) { + t.Fatal("expected different receipts to compare unequal") + } + right[0].ParseCoverage.Status = model.ParseCoverageClean + if !parseCoverageEqual(left, right) { + t.Fatal("expected identical receipts to compare equal") + } +} + +func captureIndexCoverageOutput(t *testing.T, args []string) ([]byte, error) { + t.Helper() + originalStdout := os.Stdout + readPipe, writePipe, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe failed: %v", err) + } + os.Stdout = writePipe + defer func() { os.Stdout = originalStdout }() + + runErr := runIndexCoverage(args) + _ = writePipe.Close() + var output bytes.Buffer + if _, err := output.ReadFrom(readPipe); err != nil { + t.Fatalf("ReadFrom failed: %v", err) + } + _ = readPipe.Close() + return output.Bytes(), runErr +} diff --git a/cmd/canopy/report_cmd.go b/cmd/canopy/report_cmd.go index 9d0ef2b..3dbe056 100644 --- a/cmd/canopy/report_cmd.go +++ b/cmd/canopy/report_cmd.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "m31labs.dev/canopy/internal/deps" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/boundaries" "m31labs.dev/canopy/pkg/capa" "m31labs.dev/canopy/pkg/complexity" @@ -27,10 +28,11 @@ import ( // Report is the top-level executive summary structure produced by `gts analyze report`. type Report struct { // Codebase overview - Files int `json:"files"` - Languages map[string]int `json:"languages"` - TotalSymbols int `json:"total_symbols"` - GeneratedPct int `json:"generated_pct"` + Files int `json:"files"` + Languages map[string]int `json:"languages"` + TotalSymbols int `json:"total_symbols"` + GeneratedPct int `json:"generated_pct"` + ParserHealth indexcoverage.Health `json:"parser_health"` // Complexity FunctionCount int `json:"function_count"` @@ -39,11 +41,11 @@ type Report struct { CognitiveMax int `json:"cognitive_max"` // Architecture - BoundaryViolations int `json:"boundary_violations"` - ImportCycles int `json:"import_cycles"` - AvgInstability float64 `json:"avg_instability,omitempty"` + BoundaryViolations int `json:"boundary_violations"` + ImportCycles int `json:"import_cycles"` + AvgInstability float64 `json:"avg_instability,omitempty"` MaxDistance float64 `json:"max_distance,omitempty"` - MaxLCOM int `json:"max_lcom,omitempty"` + MaxLCOM int `json:"max_lcom,omitempty"` WorstCouplingPackages []string `json:"worst_coupling_packages,omitempty"` // Type Health @@ -66,8 +68,8 @@ type Report struct { TopRiskFunctions []string `json:"top_risk_functions,omitempty"` // Structural Smells - SmellsTotal int `json:"smells_total"` - SmellErrors int `json:"smell_errors"` + SmellsTotal int `json:"smells_total"` + SmellErrors int `json:"smell_errors"` SmellWarnings int `json:"smell_warnings"` // Team breakdown (only when --by-team is set) @@ -84,13 +86,13 @@ type HotspotEntry struct { // TeamMetrics holds per-team breakdown of report metrics. type TeamMetrics struct { - Files int `json:"files"` - Functions int `json:"functions"` - CyclomaticMax int `json:"cyclomatic_max"` - CognitiveMax int `json:"cognitive_max"` - DeadFunctions int `json:"dead_functions"` + Files int `json:"files"` + Functions int `json:"functions"` + CyclomaticMax int `json:"cyclomatic_max"` + CognitiveMax int `json:"cognitive_max"` + DeadFunctions int `json:"dead_functions"` BoundaryViolations int `json:"boundary_violations"` - Capabilities int `json:"capabilities"` + Capabilities int `json:"capabilities"` } // ownerRule maps a path pattern to a team name, from CODEOWNERS or .canopyowners. @@ -155,6 +157,10 @@ Examples: if totalFiles > 0 { rpt.GeneratedPct = genFiles * 100 / totalFiles } + parserHealth, parserHealthErr := indexcoverage.BuildHealth(idx, 5) + if parserHealthErr == nil { + rpt.ParserHealth = parserHealth + } // --- Complexity --- complexityReport, complexityErr := complexity.Analyze(analysisIdx, analysisIdx.Root, complexity.Options{}) @@ -387,8 +393,8 @@ Examples: case "json": if delta != nil { return emitJSON(struct { - Current Report `json:"current"` - Baseline Report `json:"baseline"` + Current Report `json:"current"` + Baseline Report `json:"baseline"` }{ Current: rpt, Baseline: *delta, @@ -439,6 +445,30 @@ func printMarkdownReport(rpt Report, delta *Report, target string) { } fmt.Println() + // Parser Health + fmt.Println("## Parser Health") + fmt.Printf("- Status: %s\n", rpt.ParserHealth.Status) + fmt.Printf("- %d clean, %d partial, %d stopped, %d unknown, %d generated\n", + rpt.ParserHealth.Summary.Clean, + rpt.ParserHealth.Summary.Partial, + rpt.ParserHealth.Summary.Stopped, + rpt.ParserHealth.Summary.Unknown, + rpt.ParserHealth.Summary.Generated) + fmt.Printf("- %d gaps, %d parse errors\n", rpt.ParserHealth.Summary.Gaps, rpt.ParserHealth.Summary.ParseErrors) + fmt.Printf("- %d untrusted files\n", rpt.ParserHealth.Summary.Untrusted) + if len(rpt.ParserHealth.IssueFiles) > 0 { + fmt.Println("- Files that need attention:") + for _, file := range rpt.ParserHealth.IssueFiles { + fmt.Printf(" - %s (%s)\n", file.Path, file.Coverage.Status) + } + } + if delta != nil { + currentIssues := parserHealthIssueCount(rpt.ParserHealth) + baselineIssues := parserHealthIssueCount(delta.ParserHealth) + printDelta("parser issue files", currentIssues, baselineIssues) + } + fmt.Println() + // Complexity fmt.Println("## Complexity") fmt.Printf("- Max cyclomatic: %d (p90: %d)\n", rpt.CyclomaticMax, rpt.CyclomaticP90) @@ -608,6 +638,10 @@ func buildCompareReport(ref, target string, cmd *cobra.Command) (*Report, error) if totalFiles > 0 { rpt.GeneratedPct = genFiles * 100 / totalFiles } + parserHealth, parserHealthErr := indexcoverage.BuildHealth(baseIdx, 5) + if parserHealthErr == nil { + rpt.ParserHealth = parserHealth + } // Complexity complexityReport, complexityErr := complexity.Analyze(baseAnalysisIdx, baseAnalysisIdx.Root, complexity.Options{}) @@ -676,6 +710,10 @@ func buildCompareReport(ref, target string, cmd *cobra.Command) (*Report, error) return &rpt, nil } +func parserHealthIssueCount(health indexcoverage.Health) int { + return health.Summary.Untrusted +} + // newGitCmd creates an exec.Cmd for git operations in the given directory. func newGitCmd(dir string, gitArgs ...string) *exec.Cmd { cmd := exec.Command("git", gitArgs...) diff --git a/cmd/canopy/review_cmd.go b/cmd/canopy/review_cmd.go index 2b3a4c9..f1740e3 100644 --- a/cmd/canopy/review_cmd.go +++ b/cmd/canopy/review_cmd.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "m31labs.dev/canopy/internal/deps" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/boundaries" "m31labs.dev/canopy/pkg/capa" "m31labs.dev/canopy/pkg/complexity" @@ -41,6 +42,7 @@ type reviewReport struct { BoundaryIssues []boundaries.Violation `json:"boundary_issues,omitempty"` NewCapabilities []reviewCapaMatch `json:"new_capabilities,omitempty"` BlastRadius int `json:"blast_radius"` + ParserHealth *indexcoverage.Health `json:"parser_health,omitempty"` } func newReviewCmd() *cobra.Command { @@ -91,6 +93,7 @@ func newReviewCmd() *cobra.Command { if err != nil { return err } + parserHealth, parserHealthErr := indexcoverage.BuildHealthForPaths(idx, changed, 5) idx = applyGeneratedFilter(cmd, idx) report := reviewReport{ @@ -102,6 +105,9 @@ func newReviewCmd() *cobra.Command { if changedScoped { report.IndexScope = "changed" } + if parserHealthErr == nil { + report.ParserHealth = &parserHealth + } var ( reviewGraph xref.Graph reviewGraphReady bool @@ -165,6 +171,7 @@ func newReviewCmd() *cobra.Command { rules := capa.BuiltinRules() changedIdx := *idx changedIdx.Files = nil + changedIdx.Errors = nil for _, f := range idx.Files { if changedSet[f.Path] { changedIdx.Files = append(changedIdx.Files, f) @@ -204,7 +211,17 @@ func newReviewCmd() *cobra.Command { } // Text output. - fmt.Printf("review: base=%s changed_files=%d index_scope=%s blast_radius=%d\n", report.Base, report.ChangedFiles, report.IndexScope, report.BlastRadius) + fmt.Printf("review: base=%s changed_files=%d index_scope=%s blast_radius=%d", report.Base, report.ChangedFiles, report.IndexScope, report.BlastRadius) + if report.ParserHealth != nil { + fmt.Printf(" parser_health=%s", report.ParserHealth.Status) + } + fmt.Println() + if report.ParserHealth != nil && len(report.ParserHealth.IssueFiles) > 0 { + fmt.Println("\nparser issues in changed files:") + for _, file := range report.ParserHealth.IssueFiles { + fmt.Printf(" %s status=%s gaps=%d\n", file.Path, file.Coverage.Status, len(file.Coverage.Gaps)) + } + } if len(report.ComplexityDelta) > 0 { fmt.Println("\ncomplexity in changed files:") for _, cd := range report.ComplexityDelta { diff --git a/cmd/canopy/watch.go b/cmd/canopy/watch.go index 43147d4..ce18e7b 100644 --- a/cmd/canopy/watch.go +++ b/cmd/canopy/watch.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "sort" "strings" "time" @@ -290,6 +291,23 @@ func parseErrorsEqual(left, right []model.ParseError) bool { return true } +func parseCoverageEqual(left, right []model.FileSummary) bool { + if len(left) != len(right) { + return false + } + rightByPath := make(map[string]*model.ParseCoverage, len(right)) + for i := range right { + rightByPath[right[i].Path] = right[i].ParseCoverage + } + for i := range left { + rightCoverage, ok := rightByPath[left[i].Path] + if !ok || !reflect.DeepEqual(left[i].ParseCoverage, rightCoverage) { + return false + } + } + return true +} + func printIndexSummary(idx *model.Index, stats index.BuildStats, incremental bool) { if incremental { fmt.Printf( diff --git a/go.mod b/go.mod index 82ee12f..3e12000 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.7 require ( github.com/fsnotify/fsnotify v1.9.0 - github.com/odvcencio/gotreesitter v0.47.1 + github.com/odvcencio/gotreesitter v0.49.0 github.com/rasros/lx v1.3.0 github.com/spf13/cobra v1.10.2 golang.org/x/sync v0.22.0 diff --git a/go.sum b/go.sum index fb190d1..a9f3759 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db h1:v0cW/tTMrJ github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db/go.mod h1:BZyH8oba3hE/BTt2FfBDGPOHhXiKs9RFmUvvXRdzrhM= github.com/nwaples/rardecode/v2 v2.3.0 h1:CtgyxWm8ClLcSh1u4M58fOz6lmeb/j4V7KpaEi/6UtM= github.com/nwaples/rardecode/v2 v2.3.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= -github.com/odvcencio/gotreesitter v0.47.1 h1:legFCs1A3HIpBNmaGW5oYj2QexAouxTshBlGifl9HSw= -github.com/odvcencio/gotreesitter v0.47.1/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo= +github.com/odvcencio/gotreesitter v0.49.0 h1:3vjywFxk+v/Z6OGCxJi6gVy4ShwGhUUU+L3J9f+QcCk= +github.com/odvcencio/gotreesitter v0.49.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo= github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/indexcoverage/report.go b/internal/indexcoverage/report.go new file mode 100644 index 0000000..55ae87c --- /dev/null +++ b/internal/indexcoverage/report.go @@ -0,0 +1,256 @@ +// Package indexcoverage builds parser-health reports from structural indexes. +package indexcoverage + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "m31labs.dev/canopy/pkg/model" +) + +const defaultMaxFiles = 100 + +type Options struct { + IncludeClean bool + IncludeGenerated bool + MaxFiles int +} + +type Summary struct { + Clean int `json:"clean"` + Partial int `json:"partial"` + Stopped int `json:"stopped"` + Generated int `json:"generated"` + Unknown int `json:"unknown"` + ParseErrors int `json:"parse_errors"` + Gaps int `json:"gaps"` + Recovered int `json:"recovered_regions"` + IgnoredEOF int `json:"ignored_eof_missing_regions"` + Truncated int `json:"truncated_files"` + Untrusted int `json:"untrusted_files"` +} + +type File struct { + Path string `json:"path"` + Language string `json:"language,omitempty"` + Generated bool `json:"generated,omitempty"` + Coverage model.ParseCoverage `json:"coverage"` +} + +type Report struct { + Root string `json:"root"` + TotalFiles int `json:"total_files"` + Summary Summary `json:"summary"` + Files []File `json:"files,omitempty"` + Errors []model.ParseError `json:"errors,omitempty"` + DetailsTruncated bool `json:"details_truncated,omitempty"` +} + +// Health is a compact parser-confidence receipt for composite analyses. +type Health struct { + Status string `json:"status"` + TotalFiles int `json:"total_files"` + Summary Summary `json:"summary"` + IssueFiles []File `json:"issue_files,omitempty"` + DetailsTruncated bool `json:"details_truncated,omitempty"` +} + +func Build(idx *model.Index, opts Options) (Report, error) { + if idx == nil { + return Report{}, fmt.Errorf("index is nil") + } + if opts.MaxFiles <= 0 { + opts.MaxFiles = defaultMaxFiles + } + + report := Report{ + Root: idx.Root, + TotalFiles: len(idx.Files), + Errors: append([]model.ParseError(nil), idx.Errors...), + } + report.Summary.ParseErrors = len(report.Errors) + report.Summary.Untrusted = len(report.Errors) + sort.Slice(report.Errors, func(i, j int) bool { return report.Errors[i].Path < report.Errors[j].Path }) + + details := make([]File, 0) + for _, file := range idx.Files { + coverage := Normalize(file.ParseCoverage) + accumulate(&report.Summary, coverage) + if !includeDetail(coverage.Status, opts) { + continue + } + details = append(details, File{ + Path: file.Path, + Language: file.Language, + Generated: file.Generated != nil, + Coverage: coverage, + }) + } + + sort.Slice(details, func(i, j int) bool { + left := statusRank(details[i].Coverage.Status) + right := statusRank(details[j].Coverage.Status) + if left == right { + return details[i].Path < details[j].Path + } + return left < right + }) + if len(details) > opts.MaxFiles { + details = details[:opts.MaxFiles] + report.DetailsTruncated = true + } + report.Files = details + return report, nil +} + +// BuildHealth returns a compact receipt with the highest-severity status and +// a bounded list of files that need attention. +func BuildHealth(idx *model.Index, maxFiles int) (Health, error) { + report, err := Build(idx, Options{MaxFiles: maxFiles}) + if err != nil { + return Health{}, err + } + return Health{ + Status: report.Status(), + TotalFiles: report.TotalFiles, + Summary: report.Summary, + IssueFiles: report.Files, + DetailsTruncated: report.DetailsTruncated, + }, nil +} + +// BuildHealthForPaths returns parser health for the selected index paths. +func BuildHealthForPaths(idx *model.Index, paths []string, maxFiles int) (Health, error) { + if idx == nil { + return Health{}, fmt.Errorf("index is nil") + } + selected := make(map[string]struct{}, len(paths)) + for _, path := range paths { + normalized := filepath.ToSlash(filepath.Clean(strings.TrimSpace(path))) + if normalized != "" && normalized != "." { + selected[normalized] = struct{}{} + } + } + + filtered := *idx + filtered.Files = nil + filtered.Errors = nil + for _, file := range idx.Files { + if _, ok := selected[filepath.ToSlash(filepath.Clean(file.Path))]; ok { + filtered.Files = append(filtered.Files, file) + } + } + for _, parseErr := range idx.Errors { + if _, ok := selected[filepath.ToSlash(filepath.Clean(parseErr.Path))]; ok { + filtered.Errors = append(filtered.Errors, parseErr) + } + } + return BuildHealth(&filtered, maxFiles) +} + +// Status returns the highest-severity parser-health status in the report. +func (r Report) Status() string { + switch { + case r.Summary.ParseErrors > 0 || r.Summary.Stopped > 0: + return model.ParseCoverageStopped + case r.Summary.Partial > 0 || r.Summary.Truncated > 0: + return model.ParseCoveragePartial + case r.Summary.Unknown > 0 || r.TotalFiles == 0: + return model.ParseCoverageUnknown + case r.Summary.Clean > 0: + return model.ParseCoverageClean + case r.Summary.Generated > 0: + return model.ParseCoverageGenerated + default: + return model.ParseCoverageUnknown + } +} + +// StrictFailure reports whether the index contains an actionable parse gap, +// a stopped parse, a missing receipt, or a complete parse failure. +func (r Report) StrictFailure() bool { + return r.Summary.Partial > 0 || + r.Summary.Stopped > 0 || + r.Summary.Unknown > 0 || + r.Summary.ParseErrors > 0 || + r.Summary.Truncated > 0 +} + +// Normalize returns a safe copy of a file receipt with a known status. +func Normalize(input *model.ParseCoverage) model.ParseCoverage { + if input == nil { + return model.ParseCoverage{Status: model.ParseCoverageUnknown} + } + coverage := *input + coverage.Gaps = append([]model.ParseGap(nil), input.Gaps...) + coverage.Status = strings.ToLower(strings.TrimSpace(coverage.Status)) + switch coverage.Status { + case model.ParseCoverageClean, + model.ParseCoveragePartial, + model.ParseCoverageStopped, + model.ParseCoverageGenerated: + default: + coverage.Status = model.ParseCoverageUnknown + } + return coverage +} + +func accumulate(summary *Summary, coverage model.ParseCoverage) { + if summary == nil { + return + } + switch coverage.Status { + case model.ParseCoverageClean: + summary.Clean++ + case model.ParseCoveragePartial: + summary.Partial++ + case model.ParseCoverageStopped: + summary.Stopped++ + case model.ParseCoverageGenerated: + summary.Generated++ + default: + summary.Unknown++ + } + summary.Gaps += len(coverage.Gaps) + summary.Recovered += coverage.RecoveredRegions + summary.IgnoredEOF += coverage.IgnoredEOFMissingRegions + if coverage.Truncated { + summary.Truncated++ + } + if coverage.Status == model.ParseCoveragePartial || + coverage.Status == model.ParseCoverageStopped || + coverage.Status == model.ParseCoverageUnknown || + coverage.Truncated { + summary.Untrusted++ + } +} + +func includeDetail(status string, opts Options) bool { + switch status { + case model.ParseCoverageClean: + return opts.IncludeClean + case model.ParseCoverageGenerated: + return opts.IncludeGenerated + default: + return true + } +} + +func statusRank(status string) int { + switch status { + case model.ParseCoverageStopped: + return 0 + case model.ParseCoveragePartial: + return 1 + case model.ParseCoverageUnknown: + return 2 + case model.ParseCoverageGenerated: + return 3 + case model.ParseCoverageClean: + return 4 + default: + return 5 + } +} diff --git a/internal/indexcoverage/report_test.go b/internal/indexcoverage/report_test.go new file mode 100644 index 0000000..c7b1d6b --- /dev/null +++ b/internal/indexcoverage/report_test.go @@ -0,0 +1,148 @@ +package indexcoverage + +import ( + "testing" + + "m31labs.dev/canopy/pkg/model" +) + +func TestBuildAggregatesReceiptsAndDefaultsToUntrustedFiles(t *testing.T) { + idx := &model.Index{ + Root: "/repo", + Files: []model.FileSummary{ + {Path: "clean.go", Language: "go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}}, + { + Path: "broken.go", + Language: "go", + ParseCoverage: &model.ParseCoverage{ + Status: model.ParseCoveragePartial, + ErrorNodes: 1, + RecoveredRegions: 2, + Gaps: []model.ParseGap{{Kind: "error", StartLine: 3, EndLine: 4}}, + }, + }, + {Path: "legacy.go", Language: "go"}, + {Path: "generated.go", Language: "go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageGenerated}}, + }, + Errors: []model.ParseError{{Path: "failed.go", Error: "parse failure"}}, + } + + report, err := Build(idx, Options{MaxFiles: 10}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if report.Summary.Clean != 1 || report.Summary.Partial != 1 || report.Summary.Unknown != 1 || report.Summary.Generated != 1 { + t.Fatalf("unexpected summary: %+v", report.Summary) + } + if report.Summary.Gaps != 1 || report.Summary.Recovered != 2 || report.Summary.ParseErrors != 1 || report.Summary.Untrusted != 3 { + t.Fatalf("unexpected receipt totals: %+v", report.Summary) + } + if len(report.Files) != 2 || report.Files[0].Path != "broken.go" || report.Files[1].Path != "legacy.go" { + t.Fatalf("unexpected default file details: %+v", report.Files) + } + if !report.StrictFailure() { + t.Fatal("expected incomplete coverage to fail strict mode") + } +} + +func TestBuildIncludesAllReceiptsAndLimitsDetails(t *testing.T) { + idx := &model.Index{Files: []model.FileSummary{ + {Path: "a.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}}, + {Path: "b.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageGenerated}}, + {Path: "c.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoveragePartial}}, + }} + report, err := Build(idx, Options{IncludeClean: true, IncludeGenerated: true, MaxFiles: 2}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if len(report.Files) != 2 || !report.DetailsTruncated { + t.Fatalf("expected limited file details: %+v", report) + } +} + +func TestCleanReportPassesStrictMode(t *testing.T) { + report, err := Build(&model.Index{Files: []model.FileSummary{ + {Path: "main.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}}, + }}, Options{}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if report.StrictFailure() { + t.Fatalf("clean report failed strict mode: %+v", report) + } +} + +func TestTruncatedReceiptFailsStrictMode(t *testing.T) { + report, err := Build(&model.Index{Files: []model.FileSummary{ + {Path: "large.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean, Truncated: true}}, + }}, Options{}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if !report.StrictFailure() { + t.Fatalf("truncated receipt passed strict mode: %+v", report) + } +} + +func TestBuildHealthRanksIssuesAndBoundsDetails(t *testing.T) { + idx := &model.Index{Files: []model.FileSummary{ + {Path: "clean.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}}, + {Path: "partial.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoveragePartial}}, + {Path: "stopped.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageStopped}}, + }} + health, err := BuildHealth(idx, 1) + if err != nil { + t.Fatalf("BuildHealth returned error: %v", err) + } + if health.Status != model.ParseCoverageStopped { + t.Fatalf("status = %q, want %q", health.Status, model.ParseCoverageStopped) + } + if health.TotalFiles != 3 || health.Summary.Untrusted != 2 || len(health.IssueFiles) != 1 || health.IssueFiles[0].Path != "stopped.go" { + t.Fatalf("unexpected health receipt: %+v", health) + } + if !health.DetailsTruncated { + t.Fatal("expected bounded issue details to report truncation") + } +} + +func TestReportStatusUsesActionableSeverityOrder(t *testing.T) { + tests := []struct { + name string + report Report + want string + }{ + {name: "empty", report: Report{}, want: model.ParseCoverageUnknown}, + {name: "generated", report: Report{TotalFiles: 1, Summary: Summary{Generated: 1}}, want: model.ParseCoverageGenerated}, + {name: "clean", report: Report{TotalFiles: 1, Summary: Summary{Clean: 1}}, want: model.ParseCoverageClean}, + {name: "unknown", report: Report{TotalFiles: 2, Summary: Summary{Clean: 1, Unknown: 1}}, want: model.ParseCoverageUnknown}, + {name: "partial", report: Report{TotalFiles: 2, Summary: Summary{Unknown: 1, Partial: 1}}, want: model.ParseCoveragePartial}, + {name: "stopped", report: Report{TotalFiles: 2, Summary: Summary{Partial: 1, ParseErrors: 1}}, want: model.ParseCoverageStopped}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.report.Status(); got != test.want { + t.Fatalf("Status() = %q, want %q", got, test.want) + } + }) + } +} + +func TestBuildHealthForPathsFiltersFilesAndParseErrors(t *testing.T) { + idx := &model.Index{ + Files: []model.FileSummary{ + {Path: "pkg/clean.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoverageClean}}, + {Path: "pkg/partial.go", ParseCoverage: &model.ParseCoverage{Status: model.ParseCoveragePartial}}, + }, + Errors: []model.ParseError{{Path: "pkg/failed.go", Error: "failed"}}, + } + health, err := BuildHealthForPaths(idx, []string{"pkg/clean.go", "pkg/failed.go", "pkg/clean.go"}, 5) + if err != nil { + t.Fatalf("BuildHealthForPaths returned error: %v", err) + } + if health.TotalFiles != 1 || health.Summary.Clean != 1 || health.Summary.ParseErrors != 1 || health.Summary.Untrusted != 1 { + t.Fatalf("unexpected selected health: %+v", health) + } + if health.Status != model.ParseCoverageStopped { + t.Fatalf("status = %q, want %q", health.Status, model.ParseCoverageStopped) + } +} diff --git a/internal/mcp/call_context.go b/internal/mcp/call_context.go index 5fb975e..0d27b28 100644 --- a/internal/mcp/call_context.go +++ b/internal/mcp/call_context.go @@ -4,6 +4,7 @@ import ( "context" "m31labs.dev/canopy/internal/contextpack" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/contextbundle" ) @@ -11,9 +12,10 @@ import ( // Result.Content is JSON-tagged "-" (it is measured and hashed as raw // bytes, not re-encoded), so the MCP transport needs its own envelope. type bundleResponse struct { - Content string `json:"content"` - Receipt contextbundle.Receipt `json:"receipt"` - Manifest contextbundle.Manifest `json:"manifest"` + Content string `json:"content"` + Receipt contextbundle.Receipt `json:"receipt"` + Manifest contextbundle.Manifest `json:"manifest"` + ParserHealth indexcoverage.Health `json:"parser_health"` } func (s *Service) callContext(args map[string]any) (any, error) { @@ -45,10 +47,15 @@ func (s *Service) callContext(args map[string]any) (any, error) { if err != nil { return nil, err } + parserHealth, err := indexcoverage.BuildHealthForPaths(idx, manifestPaths(result.Manifest), 5) + if err != nil { + return nil, err + } return bundleResponse{ - Content: string(result.Content), - Receipt: result.Receipt, - Manifest: result.Manifest, + Content: string(result.Content), + Receipt: result.Receipt, + Manifest: result.Manifest, + ParserHealth: parserHealth, }, nil } @@ -76,6 +83,14 @@ func (s *Service) callContext(args map[string]any) (any, error) { return report, nil } +func manifestPaths(manifest contextbundle.Manifest) []string { + paths := make([]string, 0, len(manifest.Items)) + for _, item := range manifest.Items { + paths = append(paths, item.Path) + } + return paths +} + func parseSelectorsArg(args map[string]any, key string) []contextbundle.Selector { raw, ok := args[key] if !ok || raw == nil { diff --git a/internal/mcp/call_context_bundle_test.go b/internal/mcp/call_context_bundle_test.go index 04fdb00..58a98ae 100644 --- a/internal/mcp/call_context_bundle_test.go +++ b/internal/mcp/call_context_bundle_test.go @@ -6,6 +6,7 @@ import ( "testing" "m31labs.dev/canopy/internal/contextpack" + "m31labs.dev/canopy/pkg/model" ) // TestServiceCallContext_BundleMode verifies the additive gts_context @@ -54,6 +55,9 @@ func formatGreeting(name string) string { if result.Content == "" { t.Fatal("expected non-empty rendered content") } + if result.ParserHealth.Status != model.ParseCoverageClean || result.ParserHealth.TotalFiles != 1 { + t.Fatalf("expected clean parser health for selected context, got %+v", result.ParserHealth) + } } func TestServiceCallContext_BundleMode_LegacyRollback(t *testing.T) { diff --git a/internal/mcp/call_coverage.go b/internal/mcp/call_coverage.go new file mode 100644 index 0000000..470f1f7 --- /dev/null +++ b/internal/mcp/call_coverage.go @@ -0,0 +1,29 @@ +package mcp + +import ( + "fmt" + + "m31labs.dev/canopy/internal/indexcoverage" +) + +func (s *Service) callCoverage(args map[string]any) (any, error) { + target := s.stringArgOrDefault(args, "path", s.defaultRoot) + cachePath := s.stringArgOrDefault(args, "cache", s.defaultCache) + limit := intArg(args, "limit", 100) + if limit <= 0 { + return nil, fmt.Errorf("limit must be > 0") + } + + idx, err := s.loadOrBuild(cachePath, target) + if err != nil { + return nil, err + } + if generator := stringArg(args, "generator"); generator != "" { + idx = idx.FilterByGenerator(generator) + } + return indexcoverage.Build(idx, indexcoverage.Options{ + IncludeClean: boolArg(args, "include_clean", false), + IncludeGenerated: boolArg(args, "include_generated", false), + MaxFiles: limit, + }) +} diff --git a/internal/mcp/call_guardrails.go b/internal/mcp/call_guardrails.go index 76c6f84..13552a9 100644 --- a/internal/mcp/call_guardrails.go +++ b/internal/mcp/call_guardrails.go @@ -5,17 +5,20 @@ import ( "path/filepath" "strings" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/boundaries" "m31labs.dev/canopy/pkg/complexity" + "m31labs.dev/canopy/pkg/model" "m31labs.dev/canopy/pkg/xref" ) type guardrailResult struct { - File string `json:"file"` - Generated guardrailGen `json:"generated"` - Boundary guardrailBoundary `json:"boundary"` - Complexity guardrailComplex `json:"complexity"` - Warnings []string `json:"warnings"` + File string `json:"file"` + Generated guardrailGen `json:"generated"` + Boundary guardrailBoundary `json:"boundary"` + Complexity guardrailComplex `json:"complexity"` + ParserHealth model.ParseCoverage `json:"parser_health"` + Warnings []string `json:"warnings"` } type guardrailGen struct { @@ -72,8 +75,10 @@ func (s *Service) callGuardrails(args map[string]any) (any, error) { file := idx.Files[fileIdx] result := guardrailResult{ - File: file.Path, + File: file.Path, + ParserHealth: indexcoverage.Normalize(file.ParseCoverage), } + appendParserHealthWarnings(&result) // 1. Generated check. if file.Generated != nil { @@ -141,3 +146,29 @@ func (s *Service) callGuardrails(args map[string]any) (any, error) { return result, nil } + +func appendParserHealthWarnings(result *guardrailResult) { + if result == nil { + return + } + coverage := result.ParserHealth + switch coverage.Status { + case model.ParseCoverageStopped: + reason := coverage.StopReason + if reason == "" { + reason = "unspecified reason" + } + result.Warnings = append(result.Warnings, + fmt.Sprintf("parser stopped early (%s) - structural results may omit code", reason)) + case model.ParseCoveragePartial: + result.Warnings = append(result.Warnings, + fmt.Sprintf("parser recovered with %d unresolved regions - structural results may be incomplete", len(coverage.Gaps))) + case model.ParseCoverageUnknown: + result.Warnings = append(result.Warnings, + "parser coverage receipt is missing - structural results are unverified") + } + if coverage.Truncated { + result.Warnings = append(result.Warnings, + "parser gap details were truncated - inspect the coverage report before editing") + } +} diff --git a/internal/mcp/call_report.go b/internal/mcp/call_report.go index 0bc5acb..7d9fbf5 100644 --- a/internal/mcp/call_report.go +++ b/internal/mcp/call_report.go @@ -2,6 +2,7 @@ package mcp import ( "m31labs.dev/canopy/internal/deps" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/boundaries" "m31labs.dev/canopy/pkg/capa" "m31labs.dev/canopy/pkg/complexity" @@ -10,10 +11,11 @@ import ( ) type mcpReportResult struct { - Files int `json:"files"` - Languages map[string]int `json:"languages"` - TotalSymbols int `json:"total_symbols"` - GeneratedPct int `json:"generated_pct"` + Files int `json:"files"` + Languages map[string]int `json:"languages"` + TotalSymbols int `json:"total_symbols"` + GeneratedPct int `json:"generated_pct"` + ParserHealth indexcoverage.Health `json:"parser_health"` FunctionCount int `json:"function_count"` CyclomaticMax int `json:"cyclomatic_max"` @@ -67,6 +69,10 @@ func (s *Service) callReport(args map[string]any) (any, error) { if totalFiles > 0 { rpt.GeneratedPct = genFiles * 100 / totalFiles } + parserHealth, parserHealthErr := indexcoverage.BuildHealth(idx, 5) + if parserHealthErr == nil { + rpt.ParserHealth = parserHealth + } // Complexity complexityReport, complexityErr := complexity.Analyze(analysisIdx, analysisIdx.Root, complexity.Options{}) diff --git a/internal/mcp/call_review.go b/internal/mcp/call_review.go index b88d3cc..d77bca2 100644 --- a/internal/mcp/call_review.go +++ b/internal/mcp/call_review.go @@ -7,6 +7,7 @@ import ( "strings" "m31labs.dev/canopy/internal/deps" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/pkg/boundaries" "m31labs.dev/canopy/pkg/capa" "m31labs.dev/canopy/pkg/complexity" @@ -23,13 +24,14 @@ type reviewComplexityDelta struct { } type reviewReport struct { - Base string `json:"base"` - ChangedFiles int `json:"changed_files"` - Files []string `json:"files"` + Base string `json:"base"` + ChangedFiles int `json:"changed_files"` + Files []string `json:"files"` ComplexityDelta []reviewComplexityDelta `json:"complexity_delta,omitempty"` BoundaryIssues []boundaries.Violation `json:"boundary_issues,omitempty"` NewCapabilities []reviewCapaMatch `json:"new_capabilities,omitempty"` BlastRadius int `json:"blast_radius"` + ParserHealth *indexcoverage.Health `json:"parser_health,omitempty"` } type reviewCapaMatch struct { @@ -66,6 +68,7 @@ func (s *Service) callReview(args map[string]any) (any, error) { if err != nil { return nil, err } + parserHealth, parserHealthErr := indexcoverage.BuildHealthForPaths(idx, changed, 5) idx = applyGeneratedFilter(idx, boolArg(args, "include_generated", false), stringArg(args, "generator")) report := reviewReport{ @@ -73,6 +76,9 @@ func (s *Service) callReview(args map[string]any) (any, error) { ChangedFiles: len(changed), Files: changed, } + if parserHealthErr == nil { + report.ParserHealth = &parserHealth + } // 1. Complexity for changed files. compReport, compErr := complexity.Analyze(idx, idx.Root, complexity.Options{}) @@ -126,6 +132,7 @@ func (s *Service) callReview(args map[string]any) (any, error) { // Build sub-index with only changed files. changedIdx := *idx changedIdx.Files = nil + changedIdx.Errors = nil for _, f := range idx.Files { if changedSet[f.Path] { changedIdx.Files = append(changedIdx.Files, f) diff --git a/internal/mcp/service.go b/internal/mcp/service.go index e5762e3..b3a6221 100644 --- a/internal/mcp/service.go +++ b/internal/mcp/service.go @@ -119,7 +119,7 @@ func searchTools() []Tool { }, { Name: "gts_context", - Description: "Pack focused context for a file and line, or build a task-conditioned context bundle when task/mode/selectors are set", + Description: "Pack focused context, or build a task-conditioned bundle with parser health when task/mode/selectors are set", InputSchema: Schema{ Properties: map[string]Property{ "file": {Type: "string", Description: "legacy mode: file to pack context around"}, @@ -297,6 +297,20 @@ func analyzeTools() []Tool { }, }.ToMap(), }, + { + Name: "gts_coverage", + Description: "Report parser gaps, stopped parses, recovery receipts, and files with unknown parse coverage", + InputSchema: Schema{ + Properties: map[string]Property{ + "path": {Type: "string", Description: "index root path"}, + "cache": {Type: "string", Description: "index cache path"}, + "include_clean": {Type: "boolean", Description: "include clean file receipts (default: false)"}, + "include_generated": {Type: "boolean", Description: "include generated fast-path receipts (default: false)"}, + "generator": {Type: "string", Description: "filter to a specific generator or human source"}, + "limit": {Type: "integer", Description: "maximum file receipts (default: 100)"}, + }, + }.ToMap(), + }, { Name: "gts_stats", Description: "Report structural codebase metrics from an index", @@ -493,7 +507,7 @@ func analyzeTools() []Tool { }, { Name: "gts_guardrails", - Description: "Return structured advisory for a file: generated status, boundary module, complexity, fan-in warnings", + Description: "Return structured advisory for a file: parser health, generated status, boundary module, complexity, fan-in warnings", InputSchema: Schema{ Properties: map[string]Property{ "file": {Type: "string", Description: "file path to analyze (required)"}, @@ -505,7 +519,7 @@ func analyzeTools() []Tool { }, { Name: "gts_report", - Description: "Executive summary report aggregating all analyses: complexity, boundaries, import cycles, capabilities, dead code, hotspots", + Description: "Confidence-aware executive report: parser health, complexity, boundaries, import cycles, capabilities, dead code, hotspots", InputSchema: Schema{ Properties: map[string]Property{ "path": {Type: "string", Description: "index root path"}, @@ -517,7 +531,7 @@ func analyzeTools() []Tool { }, { Name: "gts_review", - Description: "Aggregate review report for changed files: complexity, boundary violations, capabilities, blast radius", + Description: "Confidence-aware review for changed files: parser health, complexity, boundaries, capabilities, blast radius", InputSchema: Schema{ Properties: map[string]Property{ "base": {Type: "string", Description: "git ref to diff against (required, e.g. main, HEAD~1)"}, @@ -693,6 +707,8 @@ func (s *Service) Call(name string, args map[string]any) (any, error) { return s.callDiff(args) case "gts_stats": return s.callStats(args) + case "gts_coverage": + return s.callCoverage(args) case "gts_files": return s.callFiles(args) case "gts_bridge": diff --git a/internal/mcp/service_test.go b/internal/mcp/service_test.go index 359df09..7273b71 100644 --- a/internal/mcp/service_test.go +++ b/internal/mcp/service_test.go @@ -13,8 +13,10 @@ import ( "m31labs.dev/canopy/internal/contextpack" "m31labs.dev/canopy/internal/deps" "m31labs.dev/canopy/internal/files" - "m31labs.dev/canopy/pkg/refactor" + "m31labs.dev/canopy/internal/indexcoverage" "m31labs.dev/canopy/internal/stats" + "m31labs.dev/canopy/pkg/model" + "m31labs.dev/canopy/pkg/refactor" "m31labs.dev/canopy/pkg/structdiff" "m31labs.dev/canopy/pkg/xref" ) @@ -30,7 +32,7 @@ func TestServiceToolsIncludesCoreRoadmapTools(t *testing.T) { for _, tool := range tools { seen[tool.Name] = true } - for _, name := range []string{"gts_grep", "gts_map", "gts_query", "gts_refs", "gts_context", "gts_scope", "gts_deps", "gts_callgraph", "gts_dead", "gts_chunk", "gts_lint", "gts_refactor", "gts_diff", "gts_stats", "gts_files", "gts_bridge"} { + for _, name := range []string{"gts_grep", "gts_map", "gts_query", "gts_refs", "gts_context", "gts_scope", "gts_deps", "gts_callgraph", "gts_dead", "gts_chunk", "gts_lint", "gts_refactor", "gts_diff", "gts_stats", "gts_coverage", "gts_files", "gts_bridge"} { if !seen[name] { t.Fatalf("expected tool %q to be present", name) } @@ -492,6 +494,42 @@ func Value() {} t.Fatalf("expected non-empty stats report, got %+v", statsReport) } + coverageRaw, err := service.Call("gts_coverage", map[string]any{"include_clean": true}) + if err != nil { + t.Fatalf("gts_coverage call failed: %v", err) + } + coverageReport, ok := coverageRaw.(indexcoverage.Report) + if !ok { + t.Fatalf("expected indexcoverage.Report, got %T", coverageRaw) + } + if coverageReport.TotalFiles == 0 || coverageReport.Summary.Clean == 0 { + t.Fatalf("expected clean coverage receipts, got %+v", coverageReport) + } + + reportRaw, err := service.Call("gts_report", map[string]any{}) + if err != nil { + t.Fatalf("gts_report call failed: %v", err) + } + report, ok := reportRaw.(mcpReportResult) + if !ok { + t.Fatalf("expected mcpReportResult, got %T", reportRaw) + } + if report.ParserHealth.Status != model.ParseCoverageClean || report.ParserHealth.TotalFiles == 0 { + t.Fatalf("expected clean parser health in report, got %+v", report.ParserHealth) + } + + guardrailsRaw, err := service.Call("gts_guardrails", map[string]any{"file": "main.go"}) + if err != nil { + t.Fatalf("gts_guardrails call failed: %v", err) + } + guardrails, ok := guardrailsRaw.(guardrailResult) + if !ok { + t.Fatalf("expected guardrailResult, got %T", guardrailsRaw) + } + if guardrails.ParserHealth.Status != model.ParseCoverageClean { + t.Fatalf("expected clean guardrail parser health, got %+v", guardrails.ParserHealth) + } + filesRaw, err := service.Call("gts_files", map[string]any{ "sort": "symbols", "top": 10, @@ -521,3 +559,27 @@ func Value() {} t.Fatalf("expected non-empty bridge report, got %+v", bridgeReport) } } + +func TestServiceGuardrailsWarnsWhenParseIsIncomplete(t *testing.T) { + tmpDir := t.TempDir() + source := []byte("package sample\nfunc Broken( {\n") + if err := os.WriteFile(filepath.Join(tmpDir, "broken.go"), source, 0o644); err != nil { + t.Fatalf("WriteFile failed: %v", err) + } + + service := NewService(tmpDir, "") + raw, err := service.Call("gts_guardrails", map[string]any{"file": "broken.go"}) + if err != nil { + t.Fatalf("gts_guardrails call failed: %v", err) + } + result, ok := raw.(guardrailResult) + if !ok { + t.Fatalf("expected guardrailResult, got %T", raw) + } + if result.ParserHealth.Status == model.ParseCoverageClean { + t.Fatalf("malformed source received a clean parser receipt: %+v", result.ParserHealth) + } + if len(result.Warnings) == 0 { + t.Fatalf("incomplete parser receipt produced no warning: %+v", result) + } +} diff --git a/pkg/index/builder.go b/pkg/index/builder.go index 8824d98..eedebbd 100644 --- a/pkg/index/builder.go +++ b/pkg/index/builder.go @@ -23,7 +23,7 @@ import ( "m31labs.dev/canopy/pkg/model" ) -const schemaVersion = "0.2.0" +const schemaVersion = "0.3.0" type Builder struct { parsers map[string]lang.Parser @@ -498,6 +498,10 @@ func (b *Builder) processWalkedFile(file grammars.ParsedFile, root string, files ) if genInfo != nil { summary = generated.FastExtractSymbols(relPath, file.Source, parser.Language()) + summary.ParseCoverage = &model.ParseCoverage{ + Status: model.ParseCoverageGenerated, + StopReason: "generated_file_fast_path", + } } else { summary, parseErr = parser.Parse(file.Path, file.Source) } @@ -769,7 +773,7 @@ func (b *Builder) buildSingleFileWithOptions(ctx context.Context, target string, func previousFilesByPath(previous *model.Index, root string) map[string]model.FileSummary { reused := map[string]model.FileSummary{} - if previous == nil { + if previous == nil || previous.Version != schemaVersion { return reused } diff --git a/pkg/index/cache_refresh.go b/pkg/index/cache_refresh.go index 8d56292..9d8a9a5 100644 --- a/pkg/index/cache_refresh.go +++ b/pkg/index/cache_refresh.go @@ -37,9 +37,10 @@ func (b *Builder) EnsureFreshCache( ctx = context.Background() } + schemaChanged := cached.Version != schemaVersion configChanged := !configHashesEqual(cached.ConfigHashes, b.configHashes) var report FreshnessReport - if !configChanged { + if !schemaChanged && !configChanged { var err error report, err = b.CheckFreshness(ctx, target, cached) if err != nil { @@ -52,7 +53,7 @@ func (b *Builder) EnsureFreshCache( base := cached status := CacheIncrementallyRefreshed - if configChanged || report.RootMismatch { + if schemaChanged || configChanged || report.RootMismatch { base = nil status = CacheFullyRebuilt } diff --git a/pkg/index/freshness_test.go b/pkg/index/freshness_test.go index 34851a0..e492050 100644 --- a/pkg/index/freshness_test.go +++ b/pkg/index/freshness_test.go @@ -176,6 +176,38 @@ func TestEnsureFreshCacheRebuildsAfterConfigChange(t *testing.T) { } } +func TestEnsureFreshCacheRebuildsAfterSchemaChange(t *testing.T) { + root := t.TempDir() + cachePath := filepath.Join(root, ".canopy", "index.json") + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc Main() {}\n") + builder, err := NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err) + } + cached, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + cached.Version = "0.2.0" + for i := range cached.Files { + cached.Files[i].ParseCoverage = nil + } + if err := Save(cachePath, cached); err != nil { + t.Fatalf("Save returned error: %v", err) + } + + refreshed, status, err := builder.EnsureFreshCache(context.Background(), root, cachePath, cached) + if err != nil { + t.Fatalf("EnsureFreshCache returned error: %v", err) + } + if status != CacheFullyRebuilt { + t.Fatalf("status = %q, want %q", status, CacheFullyRebuilt) + } + if refreshed.Version != schemaVersion || len(refreshed.Files) != 1 || refreshed.Files[0].ParseCoverage == nil { + t.Fatalf("schema rebuild did not restore parse receipts: %+v", refreshed) + } +} + func writeFreshnessSource(t *testing.T, root, name, source string) { t.Helper() if err := os.WriteFile(filepath.Join(root, name), []byte(source), 0o644); err != nil { diff --git a/pkg/index/streaming.go b/pkg/index/streaming.go index c0360a7..5a56e8c 100644 --- a/pkg/index/streaming.go +++ b/pkg/index/streaming.go @@ -144,5 +144,10 @@ func cloneFileSummary(summary model.FileSummary) model.FileSummary { g := *summary.Generated cloned.Generated = &g } + if summary.ParseCoverage != nil { + coverage := *summary.ParseCoverage + coverage.Gaps = append([]model.ParseGap(nil), summary.ParseCoverage.Gaps...) + cloned.ParseCoverage = &coverage + } return cloned } diff --git a/pkg/index/streaming_test.go b/pkg/index/streaming_test.go index 00c17c4..3f3871e 100644 --- a/pkg/index/streaming_test.go +++ b/pkg/index/streaming_test.go @@ -98,7 +98,7 @@ func TestBuildPathIncrementalWithOptions_ObserverReceivesStreamedEvents(t *testi func TestPartialIndexSnapshotTracksFilesAndErrors(t *testing.T) { base := &model.Index{ - Version: "0.2.0", + Version: "0.3.0", Root: "/repo", Files: []model.FileSummary{ {Path: "a.go", Language: "go"}, @@ -163,12 +163,29 @@ func TestPartialIndexSnapshotTracksFilesAndErrors(t *testing.T) { } } +func TestCloneFileSummaryCopiesParseCoverage(t *testing.T) { + original := model.FileSummary{ + Path: "broken.go", + ParseCoverage: &model.ParseCoverage{ + Status: model.ParseCoveragePartial, + Gaps: []model.ParseGap{{Kind: "error", StartLine: 2, EndLine: 3}}, + }, + } + cloned := cloneFileSummary(original) + cloned.ParseCoverage.Status = model.ParseCoverageClean + cloned.ParseCoverage.Gaps[0].Kind = "changed" + + if original.ParseCoverage.Status != model.ParseCoveragePartial || original.ParseCoverage.Gaps[0].Kind != "error" { + t.Fatalf("clone mutated the original receipt: %+v", original.ParseCoverage) + } +} + func TestSaveOverwritesExistingCache(t *testing.T) { tmpDir := t.TempDir() cachePath := filepath.Join(tmpDir, "nested", "index.json") first := &model.Index{ - Version: "0.2.0", + Version: "0.3.0", Root: "/first", Files: []model.FileSummary{ {Path: "a.go", Language: "go"}, @@ -176,7 +193,7 @@ func TestSaveOverwritesExistingCache(t *testing.T) { GeneratedAt: time.Unix(1, 0).UTC(), } second := &model.Index{ - Version: "0.2.0", + Version: "0.3.0", Root: "/second", Files: []model.FileSummary{ {Path: "b.go", Language: "go"}, diff --git a/pkg/lang/treesitter/coverage.go b/pkg/lang/treesitter/coverage.go new file mode 100644 index 0000000..8e146fa --- /dev/null +++ b/pkg/lang/treesitter/coverage.go @@ -0,0 +1,279 @@ +package treesitter + +import ( + "bytes" + "sort" + "unicode" + + "github.com/odvcencio/gotreesitter" + + "m31labs.dev/canopy/pkg/model" +) + +const maxParseGaps = 64 + +func buildParseCoverage(root *gotreesitter.Node, src []byte, symbols []model.Symbol, lang *gotreesitter.Language) *model.ParseCoverage { + coverage := &model.ParseCoverage{Status: model.ParseCoverageClean} + if root == nil { + if len(src) == 0 { + return coverage + } + coverage.Status = model.ParseCoverageStopped + coverage.StopReason = "missing_root" + coverage.Gaps = []model.ParseGap{sourceGap("unparsed", "source", src, 0, len(src))} + return coverage + } + + raw := collectParseGaps(root, src, coverage, lang) + coverage.Gaps = subtractRecoveredGaps(raw, symbols, coverage) + appendUnparsedSourceGaps(coverage, root, src) + if (len(coverage.Gaps) > 0 || coverage.Truncated) && coverage.Status == model.ParseCoverageClean { + coverage.Status = model.ParseCoveragePartial + } + return coverage +} + +func collectParseGaps(root *gotreesitter.Node, src []byte, coverage *model.ParseCoverage, lang *gotreesitter.Language) []model.ParseGap { + if root == nil || !root.HasErrorOrMissing() { + return nil + } + + stack := []*gotreesitter.Node{root} + gaps := make([]model.ParseGap, 0, 4) + for len(stack) > 0 { + last := len(stack) - 1 + node := stack[last] + stack = stack[:last] + if node == nil { + continue + } + + if node.IsError() || node.IsMissing() { + if isEOFTerminatorMiss(node, len(src)) { + coverage.IgnoredEOFMissingRegions++ + continue + } + if len(gaps) >= maxParseGaps { + coverage.Truncated = true + break + } + kind := "error" + if node.IsMissing() { + kind = "missing" + coverage.MissingNodes++ + } else { + coverage.ErrorNodes++ + } + gaps = append(gaps, nodeGap(kind, node, lang)) + continue + } + + for i := node.ChildCount() - 1; i >= 0; i-- { + stack = append(stack, node.Child(i)) + } + } + return gaps +} + +func isEOFTerminatorMiss(node *gotreesitter.Node, sourceLen int) bool { + if node == nil || !node.IsMissing() || sourceLen < 0 { + return false + } + return node.StartByte() == node.EndByte() && node.EndByte() == uint32(sourceLen) +} + +func nodeGap(kind string, node *gotreesitter.Node, lang *gotreesitter.Language) model.ParseGap { + if node == nil { + return model.ParseGap{Kind: kind} + } + rng := node.Range() + return model.ParseGap{ + Kind: kind, + NodeType: nodeTypeForGap(node, lang), + StartByte: rng.StartByte, + EndByte: rng.EndByte, + StartLine: int(rng.StartPoint.Row) + 1, + EndLine: int(rng.EndPoint.Row) + 1, + StartColumn: int(rng.StartPoint.Column) + 1, + EndColumn: int(rng.EndPoint.Column) + 1, + } +} + +func nodeTypeForGap(node *gotreesitter.Node, lang *gotreesitter.Language) string { + if node == nil { + return "" + } + if lang != nil { + if nodeType := node.Type(lang); nodeType != "" { + return nodeType + } + } + if node.IsError() { + return "ERROR" + } + // A missing node's grammar type is useful, but resolving it requires the + // language. The caller records "MISSING" consistently across languages. + if node.IsMissing() { + return "MISSING" + } + return "" +} + +func subtractRecoveredGaps(gaps []model.ParseGap, symbols []model.Symbol, coverage *model.ParseCoverage) []model.ParseGap { + if len(gaps) == 0 || len(symbols) == 0 { + return gaps + } + kept := make([]model.ParseGap, 0, len(gaps)) + for _, gap := range gaps { + if gapRecoveredBySymbols(gap, symbols) { + coverage.RecoveredRegions++ + continue + } + kept = append(kept, gap) + } + return kept +} + +func gapRecoveredBySymbols(gap model.ParseGap, symbols []model.Symbol) bool { + if gap.StartLine <= 0 || gap.EndLine < gap.StartLine { + return false + } + type lineSpan struct { + start int + end int + } + spans := make([]lineSpan, 0, len(symbols)) + for _, symbol := range symbols { + if symbol.StartLine < gap.StartLine || symbol.StartLine > gap.EndLine { + continue + } + end := symbol.EndLine + if end < symbol.StartLine { + end = symbol.StartLine + } + spans = append(spans, lineSpan{start: symbol.StartLine, end: end}) + } + if len(spans) == 0 { + return false + } + sort.Slice(spans, func(i, j int) bool { + if spans[i].start == spans[j].start { + return spans[i].end < spans[j].end + } + return spans[i].start < spans[j].start + }) + + coveredTo := gap.StartLine - 1 + for _, span := range spans { + if span.start > coveredTo+1 { + return false + } + if span.end > coveredTo { + coveredTo = span.end + } + } + return coveredTo >= gap.EndLine +} + +func appendUnparsedSourceGaps(coverage *model.ParseCoverage, root *gotreesitter.Node, src []byte) { + if coverage == nil || root == nil || len(src) == 0 { + return + } + start := clampByteOffset(root.StartByte(), len(src)) + end := clampByteOffset(root.EndByte(), len(src)) + if start > 0 { + if gap, ok := nonWhitespaceGap("unparsed", "source_prefix", src, 0, start); ok { + appendSyntheticGap(coverage, gap) + } + } + if end < len(src) { + if gap, ok := nonWhitespaceGap("unparsed", "source_tail", src, end, len(src)); ok { + appendSyntheticGap(coverage, gap) + } + } +} + +func appendSyntheticGap(coverage *model.ParseCoverage, gap model.ParseGap) { + for _, existing := range coverage.Gaps { + if existing.StartByte <= gap.StartByte && existing.EndByte >= gap.EndByte { + return + } + } + if len(coverage.Gaps) >= maxParseGaps { + coverage.Truncated = true + return + } + coverage.Gaps = append(coverage.Gaps, gap) + coverage.Status = model.ParseCoverageStopped + if coverage.StopReason == "" { + coverage.StopReason = "source_not_fully_parsed" + } +} + +func nonWhitespaceGap(kind, nodeType string, src []byte, start, end int) (model.ParseGap, bool) { + start, end = clampSourceRange(start, end, len(src)) + segment := src[start:end] + left := bytes.TrimLeftFunc(segment, unicode.IsSpace) + if len(left) == 0 { + return model.ParseGap{}, false + } + realStart := end - len(left) + right := bytes.TrimRightFunc(left, unicode.IsSpace) + realEnd := realStart + len(right) + return sourceGap(kind, nodeType, src, realStart, realEnd), true +} + +func sourceGap(kind, nodeType string, src []byte, start, end int) model.ParseGap { + start, end = clampSourceRange(start, end, len(src)) + startPoint := pointAtOffset(src, start) + endPoint := pointAtOffset(src, end) + return model.ParseGap{ + Kind: kind, + NodeType: nodeType, + StartByte: uint32(start), + EndByte: uint32(end), + StartLine: int(startPoint.Row) + 1, + EndLine: int(endPoint.Row) + 1, + StartColumn: int(startPoint.Column) + 1, + EndColumn: int(endPoint.Column) + 1, + } +} + +func clampSourceRange(start, end, sourceLen int) (int, int) { + if start < 0 { + start = 0 + } + if start > sourceLen { + start = sourceLen + } + if end < start { + end = start + } + if end > sourceLen { + end = sourceLen + } + return start, end +} + +func clampByteOffset(offset uint32, sourceLen int) int { + if uint64(offset) > uint64(sourceLen) { + return sourceLen + } + return int(offset) +} + +func applyTreeStopReceipt(coverage *model.ParseCoverage, tree *gotreesitter.Tree) { + if coverage == nil || tree == nil { + return + } + reason := tree.ParseStopReason() + if !tree.ParseStoppedEarly() && (reason == "" || reason == gotreesitter.ParseStopNone || reason == gotreesitter.ParseStopAccepted) { + return + } + coverage.Status = model.ParseCoverageStopped + if reason == "" || reason == gotreesitter.ParseStopNone || reason == gotreesitter.ParseStopAccepted { + coverage.StopReason = "stopped_early" + return + } + coverage.StopReason = string(reason) +} diff --git a/pkg/lang/treesitter/coverage_test.go b/pkg/lang/treesitter/coverage_test.go new file mode 100644 index 0000000..4878865 --- /dev/null +++ b/pkg/lang/treesitter/coverage_test.go @@ -0,0 +1,91 @@ +package treesitter + +import ( + "testing" + + "github.com/odvcencio/gotreesitter" + + "m31labs.dev/canopy/pkg/model" +) + +func TestParseCoverageCleanSource(t *testing.T) { + parser, err := NewParser(findEntryByExtension(t, ".go")) + if err != nil { + t.Fatalf("NewParser returned error: %v", err) + } + + summary, err := parser.Parse("main.go", []byte("package demo\n\nfunc Work() {}\n")) + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + if summary.ParseCoverage == nil { + t.Fatal("expected a parse coverage receipt") + } + if got := summary.ParseCoverage.Status; got != model.ParseCoverageClean { + t.Fatalf("coverage status = %q, want clean: %+v", got, summary.ParseCoverage) + } + if len(summary.ParseCoverage.Gaps) != 0 { + t.Fatalf("clean source has gaps: %+v", summary.ParseCoverage.Gaps) + } +} + +func TestParseCoverageReportsMalformedSource(t *testing.T) { + parser, err := NewParser(findEntryByExtension(t, ".go")) + if err != nil { + t.Fatalf("NewParser returned error: %v", err) + } + + summary, err := parser.Parse("broken.go", []byte("package demo\n\nfunc Broken( {\n")) + if err != nil { + t.Fatalf("Parse returned error: %v", err) + } + if summary.ParseCoverage == nil { + t.Fatal("expected a parse coverage receipt") + } + if summary.ParseCoverage.Status == model.ParseCoverageClean { + t.Fatalf("malformed source reported clean: %+v", summary.ParseCoverage) + } + if len(summary.ParseCoverage.Gaps) == 0 { + t.Fatalf("malformed source reported no gaps: %+v", summary.ParseCoverage) + } + gap := summary.ParseCoverage.Gaps[0] + if gap.StartLine <= 0 || gap.EndLine < gap.StartLine || gap.EndByte < gap.StartByte { + t.Fatalf("invalid gap coordinates: %+v", gap) + } +} + +func TestParseCoverageReportsNonWhitespaceTail(t *testing.T) { + source := []byte("package demo\ntrailing") + root := gotreesitter.NewLeafNode(0, true, 0, uint32(len("package demo\n")), gotreesitter.Point{}, gotreesitter.Point{Row: 1}) + coverage := buildParseCoverage(root, source, nil, nil) + + if coverage.Status != model.ParseCoverageStopped || coverage.StopReason != "source_not_fully_parsed" { + t.Fatalf("unexpected coverage: %+v", coverage) + } + if len(coverage.Gaps) != 1 || coverage.Gaps[0].NodeType != "source_tail" { + t.Fatalf("unexpected tail gaps: %+v", coverage.Gaps) + } +} + +func TestGapRecoveredBySymbolsRequiresFullLineCoverage(t *testing.T) { + gap := model.ParseGap{StartLine: 10, EndLine: 14} + if !gapRecoveredBySymbols(gap, []model.Symbol{ + {StartLine: 10, EndLine: 11}, + {StartLine: 12, EndLine: 14}, + }) { + t.Fatal("expected contiguous symbol spans to recover the gap") + } + if gapRecoveredBySymbols(gap, []model.Symbol{ + {StartLine: 10, EndLine: 11}, + {StartLine: 13, EndLine: 14}, + }) { + t.Fatal("expected an uncovered line to preserve the gap") + } +} + +func TestEmptySourceHasCleanReceipt(t *testing.T) { + coverage := buildParseCoverage(nil, nil, nil, nil) + if coverage.Status != model.ParseCoverageClean || len(coverage.Gaps) != 0 { + t.Fatalf("unexpected empty-source receipt: %+v", coverage) + } +} diff --git a/pkg/lang/treesitter/parser.go b/pkg/lang/treesitter/parser.go index 88e9944..7f2e6d7 100644 --- a/pkg/lang/treesitter/parser.go +++ b/pkg/lang/treesitter/parser.go @@ -87,11 +87,12 @@ func (p *Parser) ParseBoundTree(path string, tree *gotreesitter.BoundTree) (mode Path: path, Language: p.Language(), } - if tree == nil || tree.RootNode() == nil { + if tree == nil { return summary, nil } src := tree.Source() - if len(src) == 0 { + if tree.RootNode() == nil { + summary.ParseCoverage = buildParseCoverage(nil, src, nil, p.lang) return summary, nil } return p.buildSummaryFromRoot(path, src, tree.RootNode()), nil @@ -104,6 +105,7 @@ func (p *Parser) ParseWithTree(path string, src []byte) (model.FileSummary, *got Language: p.Language(), } if len(src) == 0 { + summary.ParseCoverage = buildParseCoverage(nil, src, nil, p.lang) return summary, gotreesitter.NewTree(nil, src, p.lang), nil } @@ -112,6 +114,8 @@ func (p *Parser) ParseWithTree(path string, src []byte) (model.FileSummary, *got return summary, nil, fmt.Errorf("parse %s: %w", path, err) } if tree == nil || tree.RootNode() == nil { + summary.ParseCoverage = buildParseCoverage(nil, src, nil, p.lang) + applyTreeStopReceipt(summary.ParseCoverage, tree) return summary, tree, nil } @@ -125,6 +129,7 @@ func (p *Parser) ParseIncrementalWithTree(path string, src, oldSrc []byte, oldTr Language: p.Language(), } if len(src) == 0 { + summary.ParseCoverage = buildParseCoverage(nil, src, nil, p.lang) return summary, gotreesitter.NewTree(nil, src, p.lang), nil } @@ -174,6 +179,8 @@ func (p *Parser) parseIncrementalTree(path string, src []byte, oldTree *gotreesi return summary, nil, fmt.Errorf("incremental parse %s: %w", path, err) } if tree == nil || tree.RootNode() == nil { + summary.ParseCoverage = buildParseCoverage(nil, src, nil, p.lang) + applyTreeStopReceipt(summary.ParseCoverage, tree) return summary, tree, nil } return p.buildSummaryFromTree(path, src, tree), tree, nil @@ -186,7 +193,9 @@ func (p *Parser) buildSummaryFromTree(path string, src []byte, tree *gotreesitte Language: p.Language(), } } - return p.buildSummaryFromRoot(path, src, tree.RootNode()) + summary := p.buildSummaryFromRoot(path, src, tree.RootNode()) + applyTreeStopReceipt(summary.ParseCoverage, tree) + return summary } func (p *Parser) buildSummaryFromRoot(path string, src []byte, root *gotreesitter.Node) model.FileSummary { @@ -198,6 +207,7 @@ func (p *Parser) buildSummaryFromRoot(path string, src []byte, root *gotreesitte summary.Imports = p.extractImports(root, src) summary.Symbols = p.extractSymbols(src, root, tags) summary.References = p.extractReferences(tags) + summary.ParseCoverage = buildParseCoverage(root, src, summary.Symbols, p.lang) return summary } diff --git a/pkg/model/model.go b/pkg/model/model.go index b576a89..9e3d3cd 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -36,6 +36,52 @@ type GeneratedInfo struct { Marker string `json:"marker,omitempty"` // the actual matched text } +const ( + // ParseCoverageClean means the parser reported no actionable syntax gap. + // It is a detection result, not a proof that every construct was extracted. + ParseCoverageClean = "clean" + // ParseCoveragePartial means the parser recovered a tree with one or more + // actionable ERROR, MISSING, or unparsed source regions. + ParseCoveragePartial = "partial" + // ParseCoverageStopped means parsing stopped before the complete source was + // accepted, for example because of a parser limit or an unparsed source tail. + ParseCoverageStopped = "stopped" + // ParseCoverageGenerated means Canopy intentionally used its generated-file + // fast path instead of building a syntax tree. + ParseCoverageGenerated = "generated" + // ParseCoverageUnknown means an index entry predates parse receipts or came + // from a parser that does not expose them. + ParseCoverageUnknown = "unknown" +) + +// ParseGap identifies one top-most source region that the syntax tree could +// not represent cleanly. Lines and columns are one-based; byte offsets are +// zero-based and use an exclusive end. +type ParseGap struct { + Kind string `json:"kind"` + NodeType string `json:"node_type,omitempty"` + StartByte uint32 `json:"start_byte"` + EndByte uint32 `json:"end_byte"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + StartColumn int `json:"start_column"` + EndColumn int `json:"end_column"` +} + +// ParseCoverage is the parser-health receipt attached to an indexed file. +// The receipt reports known gaps and recovery decisions. A clean receipt does +// not claim that a grammar or tags query covers every source construct. +type ParseCoverage struct { + Status string `json:"status"` + StopReason string `json:"stop_reason,omitempty"` + ErrorNodes int `json:"error_nodes,omitempty"` + MissingNodes int `json:"missing_nodes,omitempty"` + RecoveredRegions int `json:"recovered_regions,omitempty"` + IgnoredEOFMissingRegions int `json:"ignored_eof_missing_regions,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Gaps []ParseGap `json:"gaps,omitempty"` +} + // FileSummary contains the structural analysis of a single source file. type FileSummary struct { Path string `json:"path"` @@ -46,6 +92,7 @@ type FileSummary struct { Symbols []Symbol `json:"symbols,omitempty"` References []Reference `json:"references,omitempty"` Generated *GeneratedInfo `json:"generated,omitempty"` + ParseCoverage *ParseCoverage `json:"parse_coverage,omitempty"` } // ParseError records a file that failed to parse.