diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 8c1b73be..0df88327 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1674,12 +1674,22 @@ the real executable route and text/JSON/YAML/fallback tests now pass with a loca fake HTTP provider, closing their earlier runtime-wrapper setup blocker. All D&D module and configuration tests, their focused vet checks, contracts/shared/helper tests, and focused native CLI tests pass. Whole-tree compilation remains pending: -CLI, framework, and integration fixtures still reference removed APIs and legacy -imports. Leaf-assembled module tests also need a proper integration-test home to +CLI and integration fixtures still reference removed APIs and legacy imports. +Leaf-assembled module tests also need a proper integration-test home to satisfy the existing import-boundary check. Dependency tidy/old-path removal, notice reconciliation, remaining fixture migration, and full validation are not complete; this stage is not marked finished. +All framework package tests and vet checks now pass; the complete pipeline +package also passes race checks. Migrated resolution, construction, reference materialization, +generated handoff, codec, source/annotation isolation, diagnostics, and session +coverage to the current contracts. Removed tests tied solely to superseded +outer retries, checkpoint provenance, manifests, and attempt-debug wrappers; +native producer, cache, assessment, and capture tests retain their replacement +behavioral coverage. Whole-tree compile-only validation still fails at the old +ConfigKit YAML import in CLI tests and removed completion/checkpoint APIs in +module integration fixtures; those blockers remain required work. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/framework/pipeline/chunk_annotations_test.go b/internal/framework/pipeline/chunk_annotations_test.go index 414f528c..e6119d5e 100644 --- a/internal/framework/pipeline/chunk_annotations_test.go +++ b/internal/framework/pipeline/chunk_annotations_test.go @@ -1,11 +1,13 @@ package pipeline import ( + "context" "encoding/json" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) { @@ -73,3 +75,46 @@ func TestDebugChunkEnvelopeClonesAnnotationScopes(t *testing.T) { t.Fatal("debugSourceChunkEnvelope() shares annotation bytes") } } + +type observingChunkValidator struct { + request contracts.ChunkValidationRequest +} + +func (*observingChunkValidator) Name() string { return "observing/chunk-validator" } +func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { + v.request = request + return contracts.ValidationResult{Approved: true}, nil +} + +func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) { + prepared := preparedConcurrentPipeline(t, 1) + plan := prepared.chunker.(*typedTestChunker).plan + plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)} + plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)} + prepared.chunker = &typedTestChunker{key: prepared.resolved.Chunk.Module, plan: plan} + validator := &observingChunkValidator{} + prepared.chunkValidators.validators = []preparedValidator{{ + resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, + chunk: validator, + }} + + if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + if len(validator.request.Chunks) != 1 { + t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks) + } + chunk := validator.request.Chunks[0] + if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" { + t.Fatalf("materialized chunk identity = %#v", chunk) + } + if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` { + t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"]) + } + if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 { + t.Fatalf("materialized metadata = %#v", chunk.Metadata) + } +} diff --git a/internal/framework/pipeline/debug_execution_test.go b/internal/framework/pipeline/debug_execution_test.go deleted file mode 100644 index 15ccf2f4..00000000 --- a/internal/framework/pipeline/debug_execution_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package pipeline - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type executionDebugClient struct { - response contracts.StructuredCompletionResponse - err error -} - -func (c executionDebugClient) CompleteStructured(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) { - return c.response, c.err -} - -func TestDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) { - providerErr := contracts.NewLLMGenerationError(503, "provider unavailable") - const content = `{"ok":true}` - for _, test := range []struct { - name string - err error - content string - debugResponse bool - }{ - {name: "provider failure without body", err: providerErr}, - {name: "canceled without body", err: context.Canceled}, - {name: "split debug response", content: content, debugResponse: true}, - {name: "split fallback response", content: content}, - } { - t.Run(test.name, func(t *testing.T) { - want := contracts.LLMExecutionSummary{ - Attempts: 2, TransientRetries: 1, Outcome: "success", - CapacityWaitMS: 1, GenerationMS: 3, RetryDelayMS: 5, DurationMS: 9, - Usage: contracts.LLMObservedTokenUsage{ - PromptTokens: contracts.LLMObservedTokenCount{Value: 7, Known: true}, - CompletionTokens: contracts.LLMObservedTokenCount{Known: true}, - }, - } - if test.err != nil { - want.Outcome = "error" - want.FailureKind = "provider_status" - want.StatusCode = 503 - want.RetriesExhausted = true - if errors.Is(test.err, context.Canceled) { - want.FailureKind = "canceled" - want.StatusCode = 0 - want.RetriesExhausted = false - } - } - execution := want - response := contracts.StructuredCompletionResponse{Execution: &execution} - if test.debugResponse { - response.Debug = &contracts.LLMDebugMaterial{Response: &contracts.LLMDebugResponse{Content: test.content}} - } else { - response.Content = json.RawMessage(test.content) - } - recorder := newCapturedDebugRecorder() - client := WithDebugLLMRecording(executionDebugClient{response, test.err}, recorder) - ctx, scope := withDebugLLMScope(context.Background(), "attempt") - got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{}) - if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want { - t.Fatalf("debug completion = (%+v, %v), want summary and original error", got, err) - } - terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"}) - if err := terminal.record(nil, test.err); !errors.Is(err, test.err) { - t.Fatalf("terminal recording error = %v", err) - } - envelope := recorder.envelope(t, "attempt/response-0001.json") - encoded, err := json.Marshal(envelope.Payload) - if err != nil { - t.Fatal(err) - } - var artifact debugLLMResponseArtifact - if err := json.Unmarshal(encoded, &artifact); err != nil { - t.Fatal(err) - } - if artifact.Execution == nil || *artifact.Execution != want { - t.Fatalf("persisted response lost execution summary: %+v", artifact) - } - if !test.debugResponse && (artifact.Fallback == nil || artifact.Fallback.Execution == nil || *artifact.Fallback.Execution != want) { - t.Fatalf("persisted fallback lost execution summary: %+v", artifact.Fallback) - } - if test.content == "" { - if artifact.ContentPath != "" { - t.Fatalf("bodyless failure has content path %q", artifact.ContentPath) - } - } else { - var compact bytes.Buffer - if artifact.ContentPath == "" || json.Compact(&compact, recorder.bytes[artifact.ContentPath]) != nil || compact.String() != test.content { - t.Fatalf("split content not preserved at %q", artifact.ContentPath) - } - if (artifact.Fallback != nil && artifact.Fallback.Content != "") || (artifact.Response != nil && artifact.Response.Content != "") { - t.Fatal("split content remains in response metadata") - } - } - refs := recorder.envelope(t, "attempt/terminal.json").LLMCalls - if len(refs) != 1 || refs[0].Execution == nil || *refs[0].Execution != want || refs[0].Error != (test.err != nil) { - t.Fatalf("terminal references = %+v", refs) - } - // Decode the persisted JSON, not just Go defaults: omitted false and - // zero fields would erase the presence-aware usage contract. - usage := envelope.Payload.(map[string]any)["execution"].(map[string]any)["usage"].(map[string]any) - unknown := usage["cached_tokens"].(map[string]any) - for _, key := range []string{"value", "known", "complete"} { - if _, ok := unknown[key]; !ok { - t.Fatalf("unknown token usage omits %q: %+v", key, unknown) - } - } - }) - } -} diff --git a/internal/framework/pipeline/debug_test.go b/internal/framework/pipeline/debug_test.go index a46ee5a4..fca86eb9 100644 --- a/internal/framework/pipeline/debug_test.go +++ b/internal/framework/pipeline/debug_test.go @@ -1,78 +1,12 @@ package pipeline import ( - "context" "encoding/json" - "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -func TestCleanDebugPathPreservesRawComponents(t *testing.T) { - for _, test := range []struct { - value string - want string - }{ - {value: "", want: "%"}, - {value: ".", want: "%2E"}, - {value: "..", want: "%2E%2E"}, - {value: "a//b", want: "a/%/b"}, - {value: "/a/", want: "%/a/%"}, - {value: " a ", want: "%20a%20"}, - } { - if got := cleanDebugPath(test.value); got != test.want { - t.Errorf("cleanDebugPath(%q) = %q, want %q", test.value, got, test.want) - } - } -} - -func TestDebugLLMPathsKeepDotIdentitiesDistinct(t *testing.T) { - recorder := newCapturedDebugRecorder() - client := WithDebugLLMRecording(attemptDebugLLM{}, recorder) - for _, test := range []struct { - stageName string - path string - }{ - {stageName: ".", path: "%2E/response-0001.json"}, - {stageName: "..", path: "%2E%2E/response-0002.json"}, - } { - if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{StageName: test.stageName}, nil); err != nil { - t.Fatalf("CompleteStructured(%q): %v", test.stageName, err) - } - if !recorder.has(test.path) { - t.Errorf("debug artifact %q was not written; names = %#v", test.path, recorder.names()) - } - } -} - -func TestDebugCompletionRequestOmitsCorrectionContent(t *testing.T) { - const assistantResponse = `{"secret":"assistant response"}` - const userGuidance = "secret user guidance" - correction, err := contracts.NewSemanticCorrection([]byte(assistantResponse), userGuidance) - if err != nil { - t.Fatalf("NewSemanticCorrection() error = %v", err) - } - summary := debugCompletionRequest(contracts.StructuredCompletionRequest{ - Inputs: contracts.LLMInputSet{"source": contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")}, - Vars: map[string]any{"custom": "value"}, - Correction: correction, - }) - encoded, err := json.Marshal(summary) - if err != nil { - t.Fatalf("marshal completion summary: %v", err) - } - for _, secret := range []string{assistantResponse, userGuidance} { - if strings.Contains(string(encoded), secret) { - t.Fatalf("debug completion summary leaked %q: %s", secret, encoded) - } - } - if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil { - t.Fatalf("debug completion summary = %#v, want counts and correction metadata", summary) - } -} - func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) { doc := validSourceDocument() envelope := debugSourceDocumentEnvelope(doc) @@ -95,35 +29,6 @@ func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) { } } -func TestDebugSourceUnitsRedactSecrets(t *testing.T) { - units := []source.SourceUnit{{ - ID: 1, - Kind: "paragraph", - Text: "application text Bearer secretvalue sk-secretvalue", - Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, - Metadata: map[string]any{"api_key": "sk-secretvalue"}, - }} - - got := cloneSourceUnitsForDebug(units) - if len(got) != 1 { - t.Fatalf("debug unit count = %d, want 1", len(got)) - } - if !strings.Contains(got[0].Text, "application text") { - t.Fatalf("debug unit text = %q, want application content retained", got[0].Text) - } - for _, forbidden := range []string{"secretvalue", "sk-secretvalue"} { - if strings.Contains(got[0].Text, forbidden) { - t.Fatalf("debug unit text contains %q: %q", forbidden, got[0].Text) - } - } - if got, want := got[0].Metadata["api_key"], "[REDACTED]"; got != want { - t.Fatalf("debug unit metadata api_key = %#v, want %q", got, want) - } - if units[0].Text != "application text Bearer secretvalue sk-secretvalue" { - t.Fatalf("source unit text was mutated: %q", units[0].Text) - } -} - func TestDebugSourceChunkPreservesReference(t *testing.T) { doc := validSourceDocument() chunk := source.Chunk{ diff --git a/internal/framework/pipeline/diagnostics_test.go b/internal/framework/pipeline/diagnostics_test.go index 3b895499..046df6a0 100644 --- a/internal/framework/pipeline/diagnostics_test.go +++ b/internal/framework/pipeline/diagnostics_test.go @@ -1,48 +1,19 @@ package pipeline import ( - "context" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -func TestTerminalDiagnosticGroupsDiscardSupersededAttemptDiagnostics(t *testing.T) { - terminal, err := runProducerAttempts(context.Background(), producerAttemptConfig{Retries: 1, Policy: DefaultValidationPolicy()}, func(_ context.Context, request producerAttemptRequest) (producerAttemptOutput, error) { - if request.Number == 1 { - return producerAttemptOutput{Value: "first", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded")}, Candidate: attemptCandidate(t, "first")}, nil - } - return producerAttemptOutput{Value: "second", Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("terminal", "terminal")}}, nil - }, func(_ context.Context, output producerAttemptOutput) (validationReport, error) { - if output.Value == "first" { - return validationReport{records: []validationRecord{{validatorName: "validator", outcome: validationRejected, reasonCode: "invalid", correctionGuidance: "Correct it."}}}, nil - } - return validationReport{}, nil - }) +func TestPromoteProducerDiagnosticsPreservesValidatorOrigin(t *testing.T) { + origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells", ValidatorKey: "dnd/spells/source-relatedness"} + groups, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")}, origin, nil) if err != nil { - t.Fatalf("runProducerAttempts() error = %v", err) + t.Fatal(err) } - groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: "step", LaneID: "lane", ModuleKey: "module"}, nil) - if err != nil { - t.Fatalf("terminalDiagnosticGroups() error = %v", err) - } - if len(groups) != 1 || groups[0].Samples[0].Scope != "terminal" { - t.Fatalf("groups = %#v, want only terminal attempt diagnostics", groups) - } -} - -func TestTerminalDiagnosticGroupsPreserveValidatorOrigin(t *testing.T) { - terminal := producerAttemptTerminal{Validation: validationReport{records: []validationRecord{{ - validatorName: "dnd/spells/source-relatedness", - outcome: validationApproved, - diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("spells[0]", "not found")}, - }}}} - groups, err := terminalDiagnosticGroups(terminal, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: "extract", LaneID: "spells", ModuleKey: "dnd/spells"}, nil) - if err != nil { - t.Fatalf("terminalDiagnosticGroups() error = %v", err) - } - if len(groups) != 1 || groups[0].Origin.ValidatorKey != "dnd/spells/source-relatedness" { - t.Fatalf("groups = %#v, want validator origin", groups) + if len(groups) != 1 || groups[0].Origin != origin { + t.Fatalf("groups = %#v, want validator origin %#v", groups, origin) } } diff --git a/internal/framework/pipeline/handoff_test.go b/internal/framework/pipeline/handoff_test.go index 770493bc..49c1ebee 100644 --- a/internal/framework/pipeline/handoff_test.go +++ b/internal/framework/pipeline/handoff_test.go @@ -43,9 +43,9 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS installGeneratedTarget(&consumer.resolved.MergeReferences, StageMerge, consumer.resolved.Merge.Module) installGeneratedTarget(&consumer.resolved.NormalizeReferences, StageNormalize, consumer.resolved.Normalize.Module) - artifact, err := checkpointArtifact(prepared.Steps[0].lanes[0].typed.codec, "notes", "typed/normalize", "source", value) + artifact, err := serializeArtifact(prepared.Steps[0].lanes[0].typed.codec, value, false) if err != nil { - t.Fatalf("checkpointArtifact() error = %v", err) + t.Fatalf("serializeArtifact() error = %v", err) } input := RunInput{Prepared: prepared} input.pipeline = prepared.resolved @@ -54,23 +54,19 @@ func handoffFixture(t *testing.T, value codecNotes) (RunInput, PreparedPipelineS LaneID: "notes", NormalizerKey: "typed/normalize", SourceID: "source", - Artifact: contracts.CloneSerializedArtifact(artifact.Artifact), + Artifact: contracts.CloneSerializedArtifact(artifact), } } func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) { input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}}) - sets, provenance, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput}) + sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput}) if err != nil { t.Fatalf("buildStepReferenceSets() error = %v", err) } if got, want := len(sets), 3; got != want { t.Fatalf("reference target set count = %d, want %d", got, want) } - if got, want := len(provenance), 3; got != want { - t.Fatalf("generated provenance count = %d, want %d", got, want) - } - var first []byte for key, set := range sets { item := set.Slots["producer-output"].Items[0] @@ -91,20 +87,6 @@ func TestBuildStepReferenceSetsCanonicalizesAndClonesFanout(t *testing.T) { if got := len(step.lanes[0].resolved.ExtractReferences.ReferenceSet.Slots["producer-output"].Items); got != 0 { t.Fatalf("prepared extract reference items = %d, want zero", got) } - - encoded, err := json.Marshal(provenance) - if err != nil { - t.Fatalf("marshal generated provenance: %v", err) - } - text := string(encoded) - for _, forbidden := range []string{"first", "file://", "content_base64"} { - if strings.Contains(text, forbidden) { - t.Fatalf("generated provenance contains forbidden %q: %s", forbidden, text) - } - } - if !strings.Contains(text, "producer_pipeline_id") || !strings.Contains(text, "schema_digest") { - t.Fatalf("generated provenance = %s, want bounded producer and schema identity", text) - } } func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) { @@ -115,12 +97,12 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) { } outputs = append(outputs, producerOutput) - sets, provenance, err := buildStepReferenceSets(input, step, outputs) + sets, err := buildStepReferenceSets(input, step, outputs) if err != nil { t.Fatalf("buildStepReferenceSets() error = %v", err) } - if len(sets) != 3 || len(provenance) != 3 { - t.Fatalf("reference sets/provenance = %d/%d, want 3/3", len(sets), len(provenance)) + if len(sets) != 3 { + t.Fatalf("reference set count = %d, want 3", len(sets)) } for target, set := range sets { item := set.Slots["producer-output"].Items[0] @@ -132,7 +114,7 @@ func TestBuildStepReferenceSetsIndexesManyOutputsForFanout(t *testing.T) { func TestGeneratedReferenceAcceptsTypedEmptyCollection(t *testing.T) { input, step, producerOutput := handoffFixture(t, codecNotes{}) - sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput}) + sets, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput}) if err != nil { t.Fatalf("buildStepReferenceSets() error = %v, want nil for accepted empty collection", err) } @@ -189,7 +171,7 @@ func TestGeneratedReferenceRejectsInvalidProducerOutputsDeterministically(t *tes target := &step.lanes[0].resolved.ExtractReferences test.mutate(target, &output) } - _, _, err := buildStepReferenceSets(input, step, test.outputs(output)) + _, err := buildStepReferenceSets(input, step, test.outputs(output)) if err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("buildStepReferenceSets() error = %v, want %q", err, test.want) } @@ -208,142 +190,45 @@ func BenchmarkIndexGeneratedOutputs(b *testing.B) { } } -func TestGeneratedReferenceFingerprintChangesWithCanonicalContent(t *testing.T) { - input, step, first := handoffFixture(t, codecNotes{Items: []string{"first"}}) - firstSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{first}) - if err != nil { - t.Fatalf("build first reference set: %v", err) - } - _, _, second := handoffFixture(t, codecNotes{Items: []string{"second"}}) - secondSets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{second}) - if err != nil { - t.Fatalf("build second reference set: %v", err) - } - firstDeps := generatedReferenceDependencies(firstSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)]) - secondDeps := generatedReferenceDependencies(secondSets[keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences)]) - if reflect.DeepEqual(firstDeps, secondDeps) { - t.Fatalf("generated dependencies = %#v, want content-sensitive fingerprint", firstDeps) - } -} - -func TestGeneratedReferenceFingerprintChangesWithCanonicalSize(t *testing.T) { - input, step, producerOutput := handoffFixture(t, codecNotes{Items: []string{"first"}}) - sets, _, err := buildStepReferenceSets(input, step, []contracts.SerializedOutput{producerOutput}) - if err != nil { - t.Fatalf("build reference set: %v", err) - } - key := keyForReferenceTarget(step.lanes[0].resolved.ExtractReferences) - firstDeps := generatedReferenceDependencies(sets[key]) - changed := CloneReferenceSet(sets[key]) - item := changed.Slots["producer-output"].Items[0] - item.SizeBytes++ - changed.Slots["producer-output"] = contracts.ResolvedReferenceSlot{ - Slot: changed.Slots["producer-output"].Slot, - Items: []contracts.ReferenceItem{item}, - } - secondDeps := generatedReferenceDependencies(changed) - if reflect.DeepEqual(firstDeps, secondDeps) { - t.Fatalf("generated dependencies = %#v, want size-sensitive fingerprint", firstDeps) - } -} - func TestRunnerHandsOffAcceptedNormalizedOutputBeforeConsumerLanes(t *testing.T) { input, _, _ := handoffFixture(t, codecNotes{Items: []string{"first"}}) prepared := input.Prepared - var received contracts.ReferenceSet - prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) { + received := make(map[ModuleStage]contracts.ReferenceSet) + prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil } - prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - received = CloneReferenceSet(request.References) + prepared.Steps[1].lanes[0].typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { + received[StageExtract] = CloneReferenceSet(request.References) return erasedTypedResult{Value: codecScore{Value: 3}}, nil } - output, err := New().Run(context.Background(), input) + prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { + return erasedTypedResult{Value: codecNotes{Items: []string{"first"}}}, nil + } + prepared.Steps[1].lanes[0].typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { + received[StageMerge] = CloneReferenceSet(request.References) + return erasedTypedResult{Value: codecScore{Value: 3}}, nil + } + prepared.Steps[1].lanes[0].typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { + received[StageNormalize] = CloneReferenceSet(request.References) + return erasedTypedResult{Value: codecScore{Value: 3}}, nil + } + _, err := New().Run(context.Background(), input) if err != nil { t.Fatalf("Run() error = %v", err) } - if got, want := len(output.NormalizeOutputs), 2; got != want { - t.Fatalf("normalized output count = %d, want %d", got, want) - } - if output.NormalizeOutputs[0].StepID != "step-1" || output.NormalizeOutputs[1].StepID != "step-2" { - t.Fatalf("normalized output step IDs = %#v, want step-1 and step-2", output.NormalizeOutputs) - } - item := received.Slots["producer-output"].Items[0] - if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" { - t.Fatalf("consumer reference = %#v, want generated producer reference", item) - } - if len(output.Manifest.References) == 0 || output.Manifest.References[len(output.Manifest.References)-1].OriginType != "generated" { - t.Fatalf("manifest references = %#v, want generated provenance", output.Manifest.References) - } -} - -type handoffDependencyLoader struct { - CheckpointLoader - extract [][]CheckpointFingerprint - merge [][]CheckpointFingerprint - normalize [][]CheckpointFingerprint -} - -func (l *handoffDependencyLoader) Enabled() bool { return true } - -func (l *handoffDependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) { - l.extract = append(l.extract, append([]CheckpointFingerprint(nil), dependencies...)) - return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"} -} - -func (l *handoffDependencyLoader) Merge(_ string, _ string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) { - l.merge = append(l.merge, append([]CheckpointFingerprint(nil), dependencies...)) - return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"} -} - -func (l *handoffDependencyLoader) Normalize(_ string, _ string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) { - l.normalize = append(l.normalize, append([]CheckpointFingerprint(nil), dependencies...)) - return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"} -} - -func runHandoffWithProducerValue(t *testing.T, value codecNotes) *handoffDependencyLoader { - t.Helper() - input, _, _ := handoffFixture(t, value) - prepared := input.Prepared - prepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: value}, nil - } - prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - return erasedTypedResult{Value: value}, nil - } - prepared.Steps[1].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: codecScore{Value: 3}}, nil - } - loader := &handoffDependencyLoader{CheckpointLoader: NoopCheckpointLoader()} - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader}); err != nil { - t.Fatalf("Run() error = %v", err) - } - return loader -} - -func TestRunnerAddsGeneratedDependencyToEveryReceivingCheckpoint(t *testing.T) { - first := runHandoffWithProducerValue(t, codecNotes{Items: []string{"first"}}) - second := runHandoffWithProducerValue(t, codecNotes{Items: []string{"second"}}) - if len(first.extract) != 2 || len(first.merge) != 2 || len(first.normalize) != 2 { - t.Fatalf("checkpoint load calls = extract %d merge %d normalize %d, want two each", len(first.extract), len(first.merge), len(first.normalize)) - } - for name, calls := range map[string][][]CheckpointFingerprint{"extract": first.extract, "merge": first.merge, "normalize": first.normalize} { - if got := generatedFingerprintCount(calls[1]); got != 1 { - t.Fatalf("%s consumer dependency count = %d, want one", name, got) + for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { + set, found := received[target] + if !found || len(set.Slots["producer-output"].Items) != 1 { + t.Fatalf("%s reference set = %#v, want producer artifact", target, set) + } + item := set.Slots["producer-output"].Items[0] + if item.Origin.Type != "generated" || item.Producer.StepID != "step-1" || item.Producer.LaneID != "notes" { + t.Fatalf("%s reference = %#v, want generated producer reference", target, item) + } + var value codecNotes + if err := json.Unmarshal(item.Content, &value); err != nil || !reflect.DeepEqual(value, codecNotes{Items: []string{"first"}}) { + t.Fatalf("%s reference value = %#v, %v", target, value, err) } } - if reflect.DeepEqual(first.extract[1], second.extract[1]) || reflect.DeepEqual(first.merge[1], second.merge[1]) || reflect.DeepEqual(first.normalize[1], second.normalize[1]) { - t.Fatalf("consumer checkpoint dependencies did not change with producer content") - } -} - -func generatedFingerprintCount(dependencies []CheckpointFingerprint) int { - count := 0 - for _, dependency := range dependencies { - if strings.HasPrefix(dependency.Name, "generated-reference:") { - count++ - } - } - return count } diff --git a/internal/framework/pipeline/normalizer_registry_test.go b/internal/framework/pipeline/normalizer_registry_test.go deleted file mode 100644 index 16a3aa2f..00000000 --- a/internal/framework/pipeline/normalizer_registry_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package pipeline - -import ( - "context" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type retryingNotesNormalizer struct { - retry *contracts.NormalizeRetry -} - -func (retryingNotesNormalizer) Key() string { return "test/retry-normalize" } -func (retryingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (n retryingNotesNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) { - return contracts.TypedNormalizeResult[codecNotes]{Value: req.MergeOutput.Value, Retry: n.retry}, nil -} - -func TestNormalizerRegistryErasureClonesRetryDirective(t *testing.T) { - retry := &contracts.NormalizeRetry{ - ReasonCode: "retryable", - Message: "safe fallback available", - CorrectionGuidance: "return a complete corrected proposal", - } - registry := NewNormalizerRegistry() - if err := RegisterNormalizer(registry, ModuleSpec{Key: "test/retry-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) { - return retryingNotesNormalizer{retry: retry}, nil - }); err != nil { - t.Fatalf("RegisterNormalizer() error = %v", err) - } - entry, ok := registry.typedEntry("test/retry-normalize", "test/notes") - if !ok { - t.Fatal("typed normalizer entry missing") - } - implementation, err := entry.builder(BuildRequest{}) - if err != nil { - t.Fatalf("builder() error = %v", err) - } - result, err := entry.normalize(context.Background(), implementation, contracts.TypedNormalizeRequest[any]{MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"safe"}}}}) - if err != nil { - t.Fatalf("normalize() error = %v", err) - } - retry.Message = "mutated" - retry.CorrectionGuidance = "mutated guidance" - if result.Retry == nil || result.Retry.Message != "safe fallback available" || result.Retry.CorrectionGuidance != "return a complete corrected proposal" { - t.Fatalf("erased retry result = %#v, want independent retry data", result) - } -} diff --git a/internal/framework/pipeline/preparation_test.go b/internal/framework/pipeline/preparation_test.go index 2b33222e..01ac8fdb 100644 --- a/internal/framework/pipeline/preparation_test.go +++ b/internal/framework/pipeline/preparation_test.go @@ -132,168 +132,6 @@ func TestPrepareRetainsGeneratedSelectorsWithoutReferenceBytes(t *testing.T) { } } -func TestPreparedCheckpointFingerprintsCollectEveryComponentInStableScopeOrder(t *testing.T) { - provider := func(value string) checkpointFingerprintTestProvider { - return checkpointFingerprintTestProvider{{Name: "identity", Value: value}} - } - prepared := &PreparedPipeline{ - resolved: ResolvedPipeline{ - ID: "fingerprints", - Input: Binding("input"), - Chunk: Binding("chunk"), - Output: Binding("output"), - }, - input: provider("input"), - chunker: provider("chunk"), - chunkValidators: preparedValidatorChain{validators: []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("chunk-validator"), Target: ValidatorTargetChunk}, - chunk: provider("chunk-validator"), - }}}, - output: provider("output"), - Steps: []PreparedPipelineStep{{ID: "default"}}, - } - lane := preparedLaneExecutor{ - resolved: ResolvedArtifactLane{ - ID: "spells", - Extract: Binding("extract"), - Merge: Binding("merge"), - Normalize: Binding("normalize"), - }, - typed: &preparedTypedLane{ - extractor: provider("extract"), - merger: provider("merge"), - normalizer: provider("normalize"), - }, - extractValidators: fingerprintTestValidatorChain("extract-validator", provider("extract-validator")), - mergeValidators: fingerprintTestValidatorChain("merge-validator", provider("merge-validator")), - normalizeValidators: fingerprintTestValidatorChain("normalize-validator", provider("normalize-validator")), - } - prepared.Steps[0].lanes = []preparedLaneExecutor{lane} - - first, err := collectPreparedCheckpointFingerprints(prepared) - if err != nil { - t.Fatalf("collectPreparedCheckpointFingerprints() error = %v, want nil", err) - } - second, err := collectPreparedCheckpointFingerprints(prepared) - if err != nil { - t.Fatalf("second collection error = %v, want nil", err) - } - if !reflect.DeepEqual(first, second) { - t.Fatalf("fingerprints are not deterministic: %#v / %#v", first, second) - } - if len(first) != 10 { - t.Fatalf("fingerprints = %#v, want all 10 prepared components", first) - } - for index := 1; index < len(first); index++ { - if first[index-1].Name >= first[index].Name { - t.Fatalf("fingerprints are not sorted: %#v", first) - } - } - prepared.checkpointFingerprints = first - copy := prepared.CheckpointFingerprints() - copy[0].Value = "mutated" - if reflect.DeepEqual(copy, prepared.CheckpointFingerprints()) { - t.Fatal("CheckpointFingerprints() exposed mutable backing storage") - } -} - -func TestPreparedCheckpointFingerprintsValidateProviderValues(t *testing.T) { - tests := []struct { - name string - fingerprints []CheckpointFingerprint - want string - }{ - {name: "empty name", fingerprints: []CheckpointFingerprint{{Value: "value"}}, want: "non-empty name and value"}, - {name: "empty value", fingerprints: []CheckpointFingerprint{{Name: "name"}}, want: "non-empty name and value"}, - {name: "duplicate after trimming", fingerprints: []CheckpointFingerprint{{Name: "same", Value: "one"}, {Name: " same ", Value: "two"}}, want: "duplicated"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - prepared := &PreparedPipeline{ - resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")}, - input: checkpointFingerprintTestProvider(test.fingerprints), - } - _, err := collectPreparedCheckpointFingerprints(prepared) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("error = %v, want substring %q", err, test.want) - } - }) - } -} - -func TestPreparedCheckpointFingerprintsRemainEmptyWithoutProviders(t *testing.T) { - registries, _ := constructionRegistries(t, nil, nil) - resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog()) - if err != nil { - t.Fatal(err) - } - prepared, err := Prepare(resolved, registries, ModuleDependencies{}) - if err != nil { - t.Fatal(err) - } - if got := prepared.CheckpointFingerprints(); len(got) != 0 { - t.Fatalf("CheckpointFingerprints() = %#v, want empty", got) - } -} - -func TestPreparedCheckpointFingerprintsScopeRepeatedValidatorsByPosition(t *testing.T) { - provider := checkpointFingerprintTestProvider{{Name: "identity", Value: "same"}} - validator := func() preparedValidator { - return preparedValidator{resolved: ResolvedValidator{Binding: Binding("configured"), Target: ValidatorTargetChunk}, chunk: provider} - } - prepared := &PreparedPipeline{ - resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")}, - chunkValidators: preparedValidatorChain{validators: []preparedValidator{validator(), validator()}}, - } - got, err := collectPreparedCheckpointFingerprints(prepared) - if err != nil { - t.Fatalf("collect repeated validators: %v", err) - } - if len(got) != 2 || got[0].Name == got[1].Name { - t.Fatalf("fingerprints = %#v, want independently scoped repeated validators", got) - } -} - -type checkpointFingerprintTestProvider []CheckpointFingerprint - -func (p checkpointFingerprintTestProvider) CheckpointFingerprints() []CheckpointFingerprint { - return append([]CheckpointFingerprint(nil), p...) -} -func (checkpointFingerprintTestProvider) Key() string { return "fingerprint-test" } -func (checkpointFingerprintTestProvider) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (checkpointFingerprintTestProvider) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) { - return typedTestDocument(), nil -} -func (checkpointFingerprintTestProvider) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { - return contracts.ChunkPlanResult{}, nil -} -func (checkpointFingerprintTestProvider) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) { - return contracts.OutputResult{}, nil -} -func (checkpointFingerprintTestProvider) Name() string { return "fingerprint-test" } -func (checkpointFingerprintTestProvider) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassDeterministic -} -func (checkpointFingerprintTestProvider) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: true}, nil -} - -type checkpointFingerprintOutput struct { - contracts.OutputEncoder - fingerprints []CheckpointFingerprint -} - -func (o checkpointFingerprintOutput) CheckpointFingerprints() []CheckpointFingerprint { - return append([]CheckpointFingerprint(nil), o.fingerprints...) -} - -func fingerprintTestValidatorChain(key string, provider checkpointFingerprintTestProvider) preparedValidatorChain { - return preparedValidatorChain{validators: []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding(key), Target: ValidatorTargetTyped}, - typed: provider, - }}} -} - func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) { var built []string var observations []constructionBuildObservation @@ -351,7 +189,7 @@ func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) { t.Fatalf("prepared extract options = %#v, want original nested value", got) } - _, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest{ + _, err = prepared.Steps[0].lanes[0].typed.extract(context.Background(), prepared.Steps[0].lanes[0].typed.extractor, contracts.TypedExtractionRequest[any]{ References: CloneReferenceSet(resolved.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet), }) if err != nil { @@ -531,9 +369,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) { wantBuilt []string }{ { - name: "missing required llm dependency", + name: "missing required engine dependency", configure: func(failure *constructionFailure) { - failure.requireExtractorLLM = true + failure.requireExtractorEngine = true }, want: `lane "artifact" extract module "extract"`, wantBuilt: []string{"input", "chunk", "validator", "extract"}, @@ -573,26 +411,9 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) { } } -func TestPrepareRejectsInvalidComponentCheckpointFingerprint(t *testing.T) { - failure := &constructionFailure{outputFingerprints: []CheckpointFingerprint{{Name: "identity"}}} - registries, input := constructionRegistries(t, nil, failure) - resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog()) - if err != nil { - t.Fatal(err) - } - _, err = Prepare(resolved, registries, ModuleDependencies{}) - if err == nil || !strings.Contains(err.Error(), "checkpoint fingerprint") || !strings.Contains(err.Error(), "output:output") { - t.Fatalf("Prepare() error = %v, want scoped checkpoint fingerprint error", err) - } - if len(input.requests) != 0 { - t.Fatalf("input Parse calls = %d, want zero", len(input.requests)) - } -} - type constructionFailure struct { - requireExtractorLLM bool - output error - outputFingerprints []CheckpointFingerprint + requireExtractorEngine bool + output error } func constructionProfile() PipelineProfile { @@ -664,8 +485,8 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con extractSpec.ArtifactKind = "test/notes" if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) { record("extract", &request) - if failure.requireExtractorLLM && request.Dependencies.LLM == nil { - return nil, errors.New("structured LLM client is required") + if failure.requireExtractorEngine && request.Dependencies.Engine == nil { + return nil, errors.New("native engine is required") } return &constructionExtractor{key: "extract", requests: &input.extractRequests}, nil }); err != nil { @@ -705,11 +526,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con if failure.output != nil { return nil, failure.output } - output := contracts.OutputEncoder(&typedTestOutput{key: "output"}) - if failure.outputFingerprints != nil { - output = checkpointFingerprintOutput{OutputEncoder: output, fingerprints: failure.outputFingerprints} - } - return output, nil + return &typedTestOutput{key: "output"}, nil }); err != nil { t.Fatal(err) } @@ -719,17 +536,17 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con type constructionInput struct { key string requests []contracts.ParseRequest - extractRequests []contracts.TypedExtractionRequest + extractRequests []contracts.TypedExtractionRequest[codecNotes] } type constructionExtractor struct { key string - requests *[]contracts.TypedExtractionRequest + requests *[]contracts.TypedExtractionRequest[codecNotes] } func (extractor *constructionExtractor) Key() string { return extractor.key } func (*constructionExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[codecNotes], error) { +func (extractor *constructionExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) { if extractor.requests != nil { *extractor.requests = append(*extractor.requests, request) } diff --git a/internal/framework/pipeline/profile_test.go b/internal/framework/pipeline/profile_test.go index 614af0f0..4a33f429 100644 --- a/internal/framework/pipeline/profile_test.go +++ b/internal/framework/pipeline/profile_test.go @@ -126,148 +126,6 @@ func TestModuleCatalogExecutionClassLooksUpRegisteredMetadata(t *testing.T) { } } -func TestResolvePipelineCarriesCorrectionProtocolsIntoPreparedMetadata(t *testing.T) { - correction := contracts.CorrectionProtocolSingleResponseV1 - catalog := newProfileCatalogWithOverrides(t, - ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}}, - ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, Requires: []string{"source"}, Provides: []string{"chunk"}}, - ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}}, - ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}}, - ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: correction, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}}, - ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}}, - ) - resolved, err := ResolvePipeline(llmProfilePipeline(), ResolveOptions{}, catalog) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - if resolved.ChunkCorrectionProtocol != correction { - t.Fatalf("ChunkCorrectionProtocol = %q, want %q", resolved.ChunkCorrectionProtocol, correction) - } - lane := resolved.Steps[0].ArtifactLanes[0] - if lane.ExtractCorrectionProtocol != correction || lane.MergeCorrectionProtocol != correction || lane.NormalizeCorrectionProtocol != correction { - t.Fatalf("lane correction protocols = %q/%q/%q, want %q", lane.ExtractCorrectionProtocol, lane.MergeCorrectionProtocol, lane.NormalizeCorrectionProtocol, correction) - } - withoutCapability := cloneResolvedPipeline(resolved) - withoutCapability.ChunkCorrectionProtocol = "" - withoutCapability.Steps[0].ArtifactLanes[0].ExtractCorrectionProtocol = "" - withoutCapability.Steps[0].ArtifactLanes[0].MergeCorrectionProtocol = "" - withoutCapability.Steps[0].ArtifactLanes[0].NormalizeCorrectionProtocol = "" - withoutCapabilityDigest, err := resolvedPipelineDigest(withoutCapability) - if err != nil { - t.Fatalf("resolvedPipelineDigest() error = %v", err) - } - if withoutCapabilityDigest == resolved.Digest { - t.Fatalf("resolved digest = %q with and without correction capability, want changed", resolved.Digest) - } - - prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{}) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - if prepared.ChunkCorrectionProtocol != correction { - t.Fatalf("prepared ChunkCorrectionProtocol = %q, want %q", prepared.ChunkCorrectionProtocol, correction) - } - preparedLane := prepared.Steps[0].ArtifactLanes[0] - if preparedLane.ExtractCorrectionProtocol != correction || preparedLane.MergeCorrectionProtocol != correction || preparedLane.NormalizeCorrectionProtocol != correction { - t.Fatalf("prepared lane correction protocols = %q/%q/%q, want %q", preparedLane.ExtractCorrectionProtocol, preparedLane.MergeCorrectionProtocol, preparedLane.NormalizeCorrectionProtocol, correction) - } -} - -func TestPrepareRequiresCorrectionCapabilityForValidatorRetries(t *testing.T) { - for _, test := range []struct { - name string - protocol contracts.CorrectionProtocol - retries int - validators bool - want string - }{ - {name: "supported", protocol: contracts.CorrectionProtocolSingleResponseV1, retries: 1, validators: true}, - {name: "unsupported", retries: 1, validators: true, want: "does not declare correction protocol"}, - {name: "no validators", retries: 1}, - {name: "no retries", validators: true}, - } { - t.Run(test.name, func(t *testing.T) { - catalog := newProfileCatalogWithOverrides(t, - ModuleSpec{Key: "llm-input", Stage: StageInput, ExecutionClass: contracts.ExecutionClassLLMBacked, Provides: []string{"source"}}, - ModuleSpec{Key: "llm-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: test.protocol, Requires: []string{"source"}, Provides: []string{"chunk"}}, - ModuleSpec{Key: "llm-extractor", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}}, - ModuleSpec{Key: "llm-merge", Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}}, - ModuleSpec{Key: "llm-normalize", Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}}, - ModuleSpec{Key: "llm-output", Stage: StageOutput, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"normalized"}, Provides: []string{"encoded"}}, - ) - if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "chunk-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.ChunkValidator, error) { - return llmProfileTestChunkValidator{key: "chunk-validator"}, nil - }); err != nil { - t.Fatal(err) - } - profile := llmProfilePipeline() - profile.Chunk.Retries = test.retries - if test.validators { - profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "chunk-validator"}}} - } - resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - _, err = Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{}) - if test.want != "" { - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("Prepare() error = %v, want %q", err, test.want) - } - return - } - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - }) - } -} - -type llmProfileTestChunkValidator struct{ key string } - -func (validator llmProfileTestChunkValidator) Name() string { return validator.key } - -func (llmProfileTestChunkValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassLLMBacked -} - -func (llmProfileTestChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: true}, nil -} - -func TestResolvePipelineValidatorRetriesRequireLLMBackedValidator(t *testing.T) { - for _, test := range []struct { - name string - class contracts.ExecutionClass - want string - }{ - {name: "deterministic validator", class: contracts.ExecutionClassDeterministic, want: "retries require an LLM-backed"}, - {name: "LLM validator", class: contracts.ExecutionClassLLMBacked}, - } { - t.Run(test.name, func(t *testing.T) { - catalog := newProfileCatalog(t) - registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "retrying-validator", ExecutionClass: test.class}) - profile := baselineProfile() - lane := profile.Artifacts["events"] - lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "retrying-validator", Retries: 1}}} - profile.Artifacts["events"] = lane - resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) - if test.want != "" { - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want) - } - return - } - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - if got := resolved.ValidatorChains[1].Validators[0].Binding.Retries; got != 1 { - t.Fatalf("validator retries = %d, want 1", got) - } - }) - } -} - func TestResolvePipelineAppliesDefaults(t *testing.T) { resolved, err := ResolvePipeline(PipelineProfile{ ID: "defaulted", @@ -552,148 +410,6 @@ func TestResolvePipelineDigestUsesEffectiveLLMProfiles(t *testing.T) { } } -func TestResolvePipelineAppliesStructuredOutputRepairAttemptsToLLMBindings(t *testing.T) { - pipelineAttempts := 2 - chunkAttempts := 1 - extractAttempts := 0 - validatorAttempts := 3 - profile := llmProfilePipeline() - profile.StructuredOutputRepairAttempts = &pipelineAttempts - profile.Chunk.StructuredOutputRepairAttempts = &chunkAttempts - lane := profile.Artifacts["events"] - lane.Extract.StructuredOutputRepairAttempts = &extractAttempts - lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "llm-validator", StructuredOutputRepairAttempts: &validatorAttempts}}} - profile.Artifacts["events"] = lane - - resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalog(t)) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - encoded, err := json.Marshal(resolved) - if err != nil { - t.Fatalf("json.Marshal(resolved) error = %v", err) - } - if !strings.Contains(string(encoded), `"structured_output_repair_attempts":2`) { - t.Fatalf("resolved JSON = %s, want effective repair policy", encoded) - } - resolvedLane := resolved.Steps[0].ArtifactLanes[0] - for _, test := range []struct { - name string - got *int - want int - }{ - {name: "input", got: resolved.Input.StructuredOutputRepairAttempts, want: pipelineAttempts}, - {name: "chunk binding", got: resolved.Chunk.StructuredOutputRepairAttempts, want: chunkAttempts}, - {name: "extract binding", got: resolvedLane.Extract.StructuredOutputRepairAttempts, want: extractAttempts}, - {name: "merge binding", got: resolvedLane.Merge.StructuredOutputRepairAttempts, want: pipelineAttempts}, - {name: "normalize binding", got: resolvedLane.Normalize.StructuredOutputRepairAttempts, want: pipelineAttempts}, - {name: "output", got: resolved.Output.StructuredOutputRepairAttempts, want: pipelineAttempts}, - } { - t.Run(test.name, func(t *testing.T) { - if test.got == nil || *test.got != test.want { - t.Fatalf("repair attempts = %v, want %d", test.got, test.want) - } - }) - } - - extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "llm-extractor") - if extractChain == nil || len(extractChain.Validators) != 1 { - t.Fatalf("resolved extract validator chain = %#v, want one validator", extractChain) - } - if got := extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != validatorAttempts { - t.Fatalf("extract validator repair attempts = %v, want %d", got, validatorAttempts) - } - chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", "llm-chunk") - if chunkChain == nil || len(chunkChain.Validators) != 1 { - t.Fatalf("resolved chunk validator chain = %#v, want one validator", chunkChain) - } - if got := chunkChain.Validators[0].Binding.StructuredOutputRepairAttempts; got == nil || *got != pipelineAttempts { - t.Fatalf("chunk validator repair attempts = %v, want %d", got, pipelineAttempts) - } - - pipelineAttempts = 1 - chunkAttempts = 2 - extractAttempts = 3 - validatorAttempts = 0 - if got := *resolved.Input.StructuredOutputRepairAttempts; got != 2 { - t.Fatalf("resolved input repair attempts aliased profile: got %d, want 2", got) - } - if got := *resolved.Chunk.StructuredOutputRepairAttempts; got != 1 { - t.Fatalf("resolved chunk repair attempts aliased profile: got %d, want 1", got) - } - if got := *resolvedLane.Extract.StructuredOutputRepairAttempts; got != 0 { - t.Fatalf("resolved extract repair attempts aliased profile: got %d, want 0", got) - } - if got := *extractChain.Validators[0].Binding.StructuredOutputRepairAttempts; got != 3 { - t.Fatalf("resolved validator repair attempts aliased profile: got %d, want 3", got) - } -} - -func TestResolvePipelineRejectsStructuredOutputRepairAttemptsOnDeterministicBindings(t *testing.T) { - for _, test := range []struct { - name string - mutate func(*PipelineProfile) - }{ - {name: "module", mutate: func(profile *PipelineProfile) { profile.Input.StructuredOutputRepairAttempts = repairAttempts(1) }}, - {name: "validator", mutate: func(profile *PipelineProfile) { - lane := profile.Artifacts["events"] - lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{{Module: "grounded", StructuredOutputRepairAttempts: repairAttempts(1)}}} - profile.Artifacts["events"] = lane - }}, - } { - t.Run(test.name, func(t *testing.T) { - profile := baselineProfile() - test.mutate(&profile) - _, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t)) - if err == nil || !strings.Contains(err.Error(), "structured_output_repair_attempts") || !strings.Contains(err.Error(), "deterministic") { - t.Fatalf("ResolvePipeline() error = %v, want deterministic repair-attempt rejection", err) - } - }) - } -} - -func TestResolvePipelineLeavesPipelineRepairAttemptsOffDeterministicBindings(t *testing.T) { - profile := baselineProfile() - profile.StructuredOutputRepairAttempts = repairAttempts(2) - - resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t)) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - if resolved.Input.StructuredOutputRepairAttempts != nil || resolved.Chunk.StructuredOutputRepairAttempts != nil || resolved.Output.StructuredOutputRepairAttempts != nil { - t.Fatalf("deterministic pipeline inherited repair attempts: %#v", resolved) - } - lane := resolved.Steps[0].ArtifactLanes[0] - if lane.Extract.StructuredOutputRepairAttempts != nil || lane.Merge.StructuredOutputRepairAttempts != nil || lane.Normalize.StructuredOutputRepairAttempts != nil { - t.Fatalf("deterministic lane inherited repair attempts: %#v", lane) - } -} - -func TestResolvePipelineDigestUsesEffectiveStructuredOutputRepairAttempts(t *testing.T) { - digests := make(map[string]string) - for _, test := range []struct { - name string - attempts *int - }{ - {name: "prompt owned", attempts: nil}, - {name: "disabled", attempts: repairAttempts(0)}, - {name: "configured", attempts: repairAttempts(1)}, - } { - t.Run(test.name, func(t *testing.T) { - profile := llmProfilePipeline() - profile.StructuredOutputRepairAttempts = test.attempts - resolved, err := ResolvePipeline(profile, ResolveOptions{}, llmProfileCatalogWithoutValidatorChains(t)) - if err != nil { - t.Fatalf("ResolvePipeline() error = %v, want nil", err) - } - digests[test.name] = resolved.Digest - }) - } - if digests["prompt owned"] == digests["disabled"] || digests["disabled"] == digests["configured"] || digests["prompt owned"] == digests["configured"] { - t.Fatalf("digests = %#v, want distinct effective repair policies", digests) - } -} - func TestResolvePipelineRecordsValidatorChains(t *testing.T) { catalog := newProfileCatalog(t) if err := catalog.ValidatorChains.Register(ValidatorChainMapping{ @@ -1811,7 +1527,6 @@ func TestResolvedPipelineDigestIncludesCompleteValidatorPolicy(t *testing.T) { value.ValidatorChains[0].Validators[0].Binding.Module = "other-validator" }}, {name: "llm profile", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.LLMProfile = "fast" }}, - {name: "retries", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.Retries++ }}, {name: "options", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.Options["threshold"] = 0.75 }}, @@ -1877,7 +1592,6 @@ func validatorDigestFixture() ResolvedPipeline { Binding: ModuleBinding{ Module: "semantic-validator", LLMProfile: "careful", - Retries: 2, Options: map[string]any{"mode": "strict", "threshold": 0.5}, References: ExternalReferenceMap(map[string]string{"rules": "rules.md", "examples": "examples.md"}), }, @@ -2129,10 +1843,6 @@ func llmProfilePipeline() PipelineProfile { } } -func repairAttempts(value int) *int { - return &value -} - func llmProfileValues(profile string) map[string]string { return map[string]string{ "input": profile, diff --git a/internal/framework/pipeline/references_test.go b/internal/framework/pipeline/references_test.go index a3054a7b..de168a49 100644 --- a/internal/framework/pipeline/references_test.go +++ b/internal/framework/pipeline/references_test.go @@ -143,17 +143,6 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) { t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI) } - provenance := ReferenceProvenance(first) - if len(provenance) != 2 { - t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance) - } - if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest { - t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0]) - } - if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest { - t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1]) - } - encoded, err := json.Marshal(first) if err != nil { t.Fatalf("json.Marshal(materialized) error = %v, want nil", err) @@ -163,7 +152,7 @@ func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) { } } -func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) { +func TestMaterializeReferencesStoresSetsForAllTargets(t *testing.T) { configDir := t.TempDir() writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text")) writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text")) @@ -215,22 +204,6 @@ func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content) } - provenance := ReferenceProvenance(materialized) - if len(provenance) != 4 { - t.Fatalf("ReferenceProvenance() = %#v, want four entries", provenance) - } - if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest { - t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0]) - } - if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest { - t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1]) - } - if provenance[2].Stage != string(StageMerge) || provenance[2].LaneID != "events" || provenance[2].SlotName != "merge_notes" || provenance[2].Digest != mergeItem.Digest { - t.Fatalf("ReferenceProvenance()[2] = %#v, want merge notes provenance", provenance[2]) - } - if provenance[3].Stage != string(StageNormalize) || provenance[3].LaneID != "events" || provenance[3].SlotName != "normalization_notes" || provenance[3].Digest != normalizeItem.Digest { - t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2]) - } } func TestMaterializeReferencesUsesLaneArtifactVariant(t *testing.T) { diff --git a/internal/framework/pipeline/runner_attempt_debug_test.go b/internal/framework/pipeline/runner_attempt_debug_test.go deleted file mode 100644 index cd3ac0c8..00000000 --- a/internal/framework/pipeline/runner_attempt_debug_test.go +++ /dev/null @@ -1,478 +0,0 @@ -package pipeline - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "sync" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type capturedDebugRecorder struct { - mu sync.Mutex - json map[string][]byte - bytes map[string][]byte - failPath string -} - -func newCapturedDebugRecorder() *capturedDebugRecorder { - return &capturedDebugRecorder{json: make(map[string][]byte), bytes: make(map[string][]byte)} -} - -func (*capturedDebugRecorder) Enabled() bool { return true } - -func (r *capturedDebugRecorder) WriteJSON(name string, payload any) error { - if name == r.failPath { - return errors.New("debug recorder failure") - } - data, err := json.Marshal(payload) - if err != nil { - return err - } - r.mu.Lock() - r.json[name] = data - r.mu.Unlock() - return nil -} - -func (r *capturedDebugRecorder) WriteBytes(name string, data []byte) error { - if name == r.failPath { - return errors.New("debug recorder failure") - } - r.mu.Lock() - r.bytes[name] = append([]byte(nil), data...) - r.mu.Unlock() - return nil -} - -func (r *capturedDebugRecorder) has(name string) bool { - r.mu.Lock() - defer r.mu.Unlock() - _, jsonOK := r.json[name] - _, bytesOK := r.bytes[name] - return jsonOK || bytesOK -} - -func (r *capturedDebugRecorder) names() []string { - r.mu.Lock() - defer r.mu.Unlock() - names := make([]string, 0, len(r.json)+len(r.bytes)) - for name := range r.json { - names = append(names, name) - } - for name := range r.bytes { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func (r *capturedDebugRecorder) envelope(t *testing.T, name string) debugTimedEnvelope { - t.Helper() - r.mu.Lock() - data := append([]byte(nil), r.json[name]...) - r.mu.Unlock() - if len(data) == 0 { - t.Fatalf("debug envelope %q was not written; names = %#v", name, r.names()) - } - var envelope debugTimedEnvelope - if err := json.Unmarshal(data, &envelope); err != nil { - t.Fatalf("decode debug envelope %q: %v", name, err) - } - return envelope -} - -type attemptDebugLLM struct{} - -func (attemptDebugLLM) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, _ any) (contracts.StructuredCompletionResponse, error) { - return contracts.StructuredCompletionResponse{ - Content: json.RawMessage(`{"accepted":true}`), - Provider: "test", - Model: "test-model", - ProfileID: request.ProfileID, - Debug: &contracts.LLMDebugMaterial{ - Prompt: &contracts.LLMDebugPrompt{PromptID: request.PromptID, Messages: []contracts.LLMDebugMessage{{Role: "user", Content: "test"}}}, - Response: &contracts.LLMDebugResponse{ - Content: `{"accepted":true}`, - PromptID: request.PromptID, - ModelName: "test-model", - }, - }, - }, nil -} - -func preparedAttemptDebugPipeline(t *testing.T) *PreparedPipeline { - return preparedAttemptDebugPipelineWithChunks(t, 1) -} - -func preparedAttemptDebugPipelineWithChunks(t *testing.T, chunkCount int) *PreparedPipeline { - t.Helper() - prepared := preparedConcurrentPipeline(t, chunkCount) - prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1] - prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1] - prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1] - prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{} - prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{} - return prepared -} - -func TestRunnerForwardsDetachedStructuredOutputRepairAttempts(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - producerAttempts := 2 - validatorAttempts := 0 - lane.resolved.Extract.StructuredOutputRepairAttempts = &producerAttempts - if len(lane.extractValidators.validators) == 0 { - t.Fatal("extract validators are empty") - } - lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts = &validatorAttempts - - var observedProducer, observedValidator *int - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - observedProducer = request.StructuredOutputRepairAttempts - return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}}, nil - }) - lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) { - observedValidator = target.structuredOutputRepairAttempts - return contracts.ValidationResult{Approved: true}, nil - } - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if observedProducer == nil || *observedProducer != producerAttempts || observedProducer == lane.resolved.Extract.StructuredOutputRepairAttempts { - t.Fatalf("producer repair attempts = %v, want detached value %d", observedProducer, producerAttempts) - } - if observedValidator == nil || *observedValidator != validatorAttempts || observedValidator == lane.extractValidators.validators[0].resolved.Binding.StructuredOutputRepairAttempts { - t.Fatalf("validator repair attempts = %v, want detached value %d", observedValidator, validatorAttempts) - } -} - -func callAttemptDebugLLM(ctx context.Context, client contracts.StructuredLLMClient, name string) error { - _, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "unscoped-" + name, PromptID: name, ProfileID: "test"}, nil) - return err -} - -func TestRunnerWritesAttemptScopedMergeAndNormalizeDebug(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - debug := newCapturedDebugRecorder() - client := WithDebugLLMRecording(attemptDebugLLM{}, debug) - prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - if err := callAttemptDebugLLM(ctx, client, "merge"); err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("merge", "merge diagnostic")}}, nil - } - prepared.Steps[0].lanes[0].typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - if err := callAttemptDebugLLM(ctx, client, "normalize"); err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{"normalized"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("normalize", "normalize diagnostic")}}, nil - } - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %d, want one", len(output.NormalizeOutputs)) - } - - wantPaths := []string{ - "merge/notes/input.json", - "merge/notes/attempt-01.json", - "merge/notes/attempt-01/prompt-0001.json", - "merge/notes/attempt-01/response-0001.json", - "merge/notes/attempt-01/response-content-0001.json", - "merge/notes/output.json", - "normalize/notes/input.json", - "normalize/notes/attempt-01.json", - "normalize/notes/attempt-01/prompt-0002.json", - "normalize/notes/attempt-01/response-0002.json", - "normalize/notes/attempt-01/response-content-0002.json", - "normalize/notes/output.json", - } - for _, name := range wantPaths { - if !debug.has(name) { - t.Errorf("debug artifact %q missing; names = %#v", name, debug.names()) - } - } - for _, name := range debug.names() { - if strings.HasPrefix(name, "unscoped-merge/") || strings.HasPrefix(name, "unscoped-normalize/") { - t.Errorf("LLM call used fallback debug path %q", name) - } - } - mergeEnvelope := debug.envelope(t, "merge/notes/attempt-01.json") - normalizeEnvelope := debug.envelope(t, "normalize/notes/attempt-01.json") - if len(mergeEnvelope.LLMCalls) != 1 || mergeEnvelope.LLMCalls[0].ResponsePath != "merge/notes/attempt-01/response-0001.json" { - t.Fatalf("merge LLM calls = %#v, want attempt-scoped call", mergeEnvelope.LLMCalls) - } - if len(normalizeEnvelope.LLMCalls) != 1 || normalizeEnvelope.LLMCalls[0].ResponsePath != "normalize/notes/attempt-01/response-0002.json" { - t.Fatalf("normalize LLM calls = %#v, want attempt-scoped call", normalizeEnvelope.LLMCalls) - } -} - -func TestRunnerRecordsDistinctRetryAttemptsAndPromotesAcceptedDiagnosticsOnly(t *testing.T) { - for _, stage := range []ModuleStage{StageMerge, StageNormalize} { - t.Run(string(stage), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - debug := newCapturedDebugRecorder() - client := WithDebugLLMRecording(attemptDebugLLM{}, debug) - lane := &prepared.Steps[0].lanes[0] - attempts := 0 - operation := func(ctx context.Context) (erasedTypedResult, error) { - attempts++ - if err := callAttemptDebugLLM(ctx, client, string(stage)); err != nil { - return erasedTypedResult{}, err - } - scope := "accepted" - if attempts == 1 { - scope = "discarded" - } - return erasedTypedResult{Value: codecNotes{Items: []string{scope}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope))}, nil - } - validatorCalls := 0 - validator := preparedValidator{ - resolved: ResolvedValidator{Binding: Binding("retry-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - validatorCalls++ - return contracts.ValidationResult{Approved: validatorCalls > 1, ReasonCode: "retry", Message: "retry candidate", CorrectionGuidance: "return an acceptable candidate"}, nil - }, - } - switch stage { - case StageMerge: - lane.resolved.Merge.Retries = 1 - lane.mergeValidators.validators = []preparedValidator{validator} - lane.typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - return operation(ctx) - } - case StageNormalize: - lane.resolved.Normalize.Retries = 1 - lane.normalizeValidators.validators = []preparedValidator{validator} - lane.typed.normalize = func(ctx context.Context, _ any, _ contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - return operation(ctx) - } - } - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - laneID := lane.resolved.ID - firstPath := fmt.Sprintf("%s/%s/attempt-01.json", stage, laneID) - secondPath := fmt.Sprintf("%s/%s/attempt-02.json", stage, laneID) - assertAttemptEnvelopeSequence(t, debug, fmt.Sprintf("%s/%s", stage, laneID), 1, 2) - first := debug.envelope(t, firstPath) - second := debug.envelope(t, secondPath) - if len(first.LLMCalls) != 1 || len(second.LLMCalls) != 1 || first.LLMCalls[0].CallID == second.LLMCalls[0].CallID { - t.Fatalf("retry LLM calls = first %#v, second %#v; want distinct calls", first.LLMCalls, second.LLMCalls) - } - if !strings.Contains(string(debug.json[firstPath]), "rejection") { - t.Fatalf("first attempt envelope = %s, want rejection", debug.json[firstPath]) - } - if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" { - t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics) - } - }) - } -} - -func TestRunnerAttemptDebugRepresentsFailuresAndFinalRejection(t *testing.T) { - tests := []struct { - name string - configure func(*PreparedPipeline) - path string - wantError string - wantBody string - attemptError bool - }{ - { - name: "merge module error", - configure: func(prepared *PreparedPipeline) { - prepared.Steps[0].lanes[0].typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - return erasedTypedResult{}, errors.New("merge exploded") - } - }, - path: "merge/notes/attempt-01.json", - wantError: "merge exploded", - attemptError: true, - }, - { - name: "normalize validator error", - configure: func(prepared *PreparedPipeline) { - prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureFailRun - prepared.Steps[0].lanes[0].normalizeValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("error-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, errors.New("validator exploded") - }, - }} - }, - path: "normalize/notes/attempt-01.json", - wantError: "validator exploded", - wantBody: "output", - }, - { - name: "merge final rejection", - configure: func(prepared *PreparedPipeline) { - prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("reject-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil - }, - }} - }, - path: "merge/notes/attempt-01.json", - wantBody: "rejection", - }, - { - name: "normalize serialization error", - configure: func(prepared *PreparedPipeline) { - prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - return erasedTypedResult{Value: "wrong artifact type"}, nil - } - }, - path: "normalize/notes/attempt-01.json", - wantError: "serialize normalize candidate", - attemptError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - debug := newCapturedDebugRecorder() - tc.configure(prepared) - output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - envelope := debug.envelope(t, tc.path) - if tc.wantError != "" { - if runErr == nil || !strings.Contains(runErr.Error(), tc.wantError) { - t.Fatalf("run error = %v, envelope error = %q; want %q", runErr, envelope.Error, tc.wantError) - } - if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) { - t.Fatalf("attempt envelope error = %q, want %q", envelope.Error, tc.wantError) - } - if !tc.attemptError && envelope.Error != "" { - t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error) - } - } else if runErr != nil { - t.Fatalf("Run() error = %v, want nil rejection outcome", runErr) - } - if tc.wantBody != "" && !strings.Contains(string(debug.json[tc.path]), tc.wantBody) { - t.Fatalf("attempt envelope = %s, want %q", debug.json[tc.path], tc.wantBody) - } - if tc.name == "merge final rejection" && len(output.Rejected) != 1 { - t.Fatalf("rejected outputs = %#v, want one", output.Rejected) - } - }) - } -} - -func TestRunnerKeepsValidatorLLMCallsOutOfModuleAttempt(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - debug := newCapturedDebugRecorder() - client := WithDebugLLMRecording(attemptDebugLLM{}, debug) - prepared.Steps[0].lanes[0].typed.merge = func(ctx context.Context, _ any, _ contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - if err := callAttemptDebugLLM(ctx, client, "merge-module"); err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{"merged"}}}, nil - } - prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) { - if err := callAttemptDebugLLM(ctx, client, "merge-validator"); err != nil { - return contracts.ValidationResult{}, err - } - return contracts.ValidationResult{Approved: true}, nil - }, - }} - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - moduleEnvelope := debug.envelope(t, "merge/notes/attempt-01.json") - validatorPath := "validate/merge/notes/typed%2Fmerge/01-llm-check-attempt-01.json" - validatorEnvelope := debug.envelope(t, validatorPath) - if len(moduleEnvelope.LLMCalls) != 1 || !strings.Contains(moduleEnvelope.LLMCalls[0].ResponsePath, "merge/notes/attempt-01/") { - t.Fatalf("module LLM calls = %#v, want module call only", moduleEnvelope.LLMCalls) - } - if len(validatorEnvelope.LLMCalls) != 1 || !strings.Contains(validatorEnvelope.LLMCalls[0].ResponsePath, "validate/merge/notes/") { - t.Fatalf("validator LLM calls = %#v, want validator call only", validatorEnvelope.LLMCalls) - } - if moduleEnvelope.LLMCalls[0].CallID == validatorEnvelope.LLMCalls[0].CallID { - t.Fatalf("module and validator envelopes reference the same call: %#v", moduleEnvelope.LLMCalls) - } -} - -type attemptReuseLoader struct { - CheckpointLoader - laneID string - merge MergeCheckpoint - normalize NormalizeCheckpoint -} - -func (l attemptReuseLoader) Enabled() bool { return true } - -func (l attemptReuseLoader) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) { - if laneID == l.laneID { - return l.merge, CheckpointDecision{Reused: true, Reason: "test reuse"} - } - return MergeCheckpoint{}, CheckpointDecision{Reason: "not found"} -} - -func (l attemptReuseLoader) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) { - if laneID == l.laneID { - return l.normalize, CheckpointDecision{Reused: true, Reason: "test reuse"} - } - return NormalizeCheckpoint{}, CheckpointDecision{Reason: "not found"} -} - -func TestRunnerCheckpointReuseDoesNotSynthesizeModuleAttempts(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := prepared.Steps[0].lanes[0] - merge, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Merge.Module, "source", codecNotes{Items: []string{"merged"}}) - if err != nil { - t.Fatalf("checkpointArtifact(merge): %v", err) - } - normalize, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Normalize.Module, "source", codecNotes{Items: []string{"normalized"}}) - if err != nil { - t.Fatalf("checkpointArtifact(normalize): %v", err) - } - loader := attemptReuseLoader{ - CheckpointLoader: NoopCheckpointLoader(), - laneID: lane.resolved.ID, - merge: MergeCheckpoint{Output: merge}, - normalize: NormalizeCheckpoint{Output: normalize}, - } - debug := newCapturedDebugRecorder() - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoint: loader}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - for _, name := range debug.names() { - if (strings.HasPrefix(name, "merge/notes/attempt-") || strings.HasPrefix(name, "normalize/notes/attempt-")) && name != "" { - t.Fatalf("checkpoint reuse synthesized module attempt artifact %q", name) - } - } - for _, name := range []string{"merge/notes/input.json", "merge/notes/output.json", "normalize/notes/input.json", "normalize/notes/output.json"} { - if !debug.has(name) { - t.Errorf("checkpoint reuse missing stage-level artifact %q", name) - } - } -} - -func TestRunnerTreatsModuleAttemptDebugWriteFailureAsFrameworkError(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - debug := newCapturedDebugRecorder() - debug.failPath = "merge/notes/attempt-01.json" - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err == nil || !strings.Contains(err.Error(), "write merge attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { - t.Fatalf("Run() error = %v, want merge attempt debug failure", err) - } -} diff --git a/internal/framework/pipeline/runner_candidate_encoding_test.go b/internal/framework/pipeline/runner_candidate_encoding_test.go index e44cf1b7..04026644 100644 --- a/internal/framework/pipeline/runner_candidate_encoding_test.go +++ b/internal/framework/pipeline/runner_candidate_encoding_test.go @@ -2,10 +2,8 @@ package pipeline import ( "context" - "encoding/base64" "encoding/json" "errors" - "fmt" "reflect" "strings" "testing" @@ -93,26 +91,6 @@ func matchingNotes(values []codecNotes, value string) int { return count } -type candidateCheckpointRecorder struct { - CheckpointRecorder - mergeSucceeded int - normalizeSucceeded int - mergeOutput CheckpointArtifact - normalizeOutput CheckpointArtifact -} - -func (r *candidateCheckpointRecorder) MergeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error { - r.mergeSucceeded++ - r.mergeOutput = cloneCheckpointArtifact(output) - return nil -} - -func (r *candidateCheckpointRecorder) NormalizeSucceeded(_ string, _ string, _ []CheckpointFingerprint, output CheckpointArtifact) error { - r.normalizeSucceeded++ - r.normalizeOutput = cloneCheckpointArtifact(output) - return nil -} - func installObservedNotesCodec(t *testing.T, prepared *PreparedPipeline, codec *observedNotesCodec) { t.Helper() registry := NewArtifactCodecRegistry() @@ -130,7 +108,7 @@ func configureCandidateOperation(prepared *PreparedPipeline, target ModuleStage, lane := &prepared.Steps[0].lanes[0] switch target { case StageExtract: - lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) { + lane.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { return erasedTypedResult{Value: value}, nil } lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { @@ -176,12 +154,7 @@ func setCandidateValidator(prepared *PreparedPipeline, target ModuleStage, appro func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) { for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { t.Run(string(target), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - if target == StageMerge { - prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - } else { - prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - } + prepared := preparedConcurrentPipeline(t, 1) prepared.Steps[0].lanes[0].extractValidators = preparedValidatorChain{} prepared.Steps[0].lanes[0].mergeValidators = preparedValidatorChain{} prepared.Steps[0].lanes[0].normalizeValidators = preparedValidatorChain{} @@ -233,8 +206,7 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) { } } - debug := newCapturedDebugRecorder() - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) + output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } @@ -251,23 +223,10 @@ func TestRunnerIsolatesTypedValidatorCandidates(t *testing.T) { } case StageNormalize: var normalized codecNotes - if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) { + if err := json.Unmarshal(output.normalizeOutputs[0].Artifact.Content, &normalized); err != nil || !reflect.DeepEqual(normalized, candidate) { t.Fatalf("normalized output = %#v, %v; want %#v", normalized, err, candidate) } } - attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target) - if target == StageExtract { - attemptPath = "extract/notes/chunk-000001/attempt-01.json" - } - payload := debug.envelope(t, attemptPath).Payload.(map[string]any) - content, err := base64.StdEncoding.DecodeString(payload["output"].(map[string]any)["content"].(map[string]any)["content_base64"].(string)) - var debugValue codecNotes - if err == nil { - err = json.Unmarshal(content, &debugValue) - } - if err != nil || !reflect.DeepEqual(debugValue, candidate) { - t.Fatalf("debug candidate = %#v, %v; want %#v", debugValue, err, candidate) - } }) } } @@ -284,7 +243,7 @@ func setCandidateValidators(prepared *PreparedPipeline, target ModuleStage, vali } func TestRunnerReportsCandidateDecodeFailure(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) + prepared := preparedConcurrentPipeline(t, 1) codec := &observedNotesCodec{candidateDecodeError: "decode-failure"} installObservedNotesCodec(t, prepared, codec) configureCandidateOperation(prepared, StageMerge, codecNotes{Items: []string{"decode-failure"}}) @@ -298,131 +257,74 @@ func TestRunnerReportsCandidateDecodeFailure(t *testing.T) { } func TestRunnerRejectsCandidatesBeforeFinalEncoding(t *testing.T) { - for _, target := range []ModuleStage{StageMerge, StageNormalize} { + for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { t.Run(string(target), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - if target == StageMerge { - prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - } else { - prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - } + prepared := preparedConcurrentPipeline(t, 1) codec := &observedNotesCodec{} installObservedNotesCodec(t, prepared, codec) configureCandidateOperation(prepared, target, codecNotes{Items: []string{"invalid"}}) setCandidateValidator(prepared, target, false) - recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder}) - if err != nil { - t.Fatalf("Run() error = %v, want validator rejection", err) - } - if len(output.Rejected) != 1 || output.Rejected[0].Stage != string(target) || output.Rejected[0].ReasonCode != "candidate_rejected" { - t.Fatalf("rejected outputs = %#v, want %s validator rejection", output.Rejected, target) + output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) + if err == nil || !strings.Contains(err.Error(), "candidate-check") || len(output.OutputFiles) != 0 { + t.Fatalf("result=%#v error=%v, want rejection without publication", output, err) } if codec.candidateCalls("invalid") != 1 || codec.finalCalls("invalid") != 0 { - t.Fatalf("invalid candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid")) - } - if target == StageMerge && recorder.mergeSucceeded != 0 { - t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded) - } - if target == StageNormalize && recorder.normalizeSucceeded != 0 { - t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded) + t.Fatalf("rejected candidate calls = candidate %d, final %d; want 1, 0", codec.candidateCalls("invalid"), codec.finalCalls("invalid")) } }) } } func TestRunnerFinalEncodesAcceptedCandidatesOnce(t *testing.T) { - for _, target := range []ModuleStage{StageMerge, StageNormalize} { + for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { t.Run(string(target), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) + prepared := preparedConcurrentPipeline(t, 1) codec := &observedNotesCodec{} installObservedNotesCodec(t, prepared, codec) value := "accepted-" + string(target) configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}}) setCandidateValidator(prepared, target, true) - recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - loader := &extractResultLoader{CheckpointLoader: NoopCheckpointLoader(), decision: CheckpointDecision{Reason: "not found"}} - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Checkpoint: loader}) + _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) if err != nil { - t.Fatalf("Run() error = %v, want nil", err) + t.Fatal(err) } if codec.candidateCalls(value) != 1 || codec.finalCalls(value) != 1 { t.Fatalf("accepted candidate calls = candidate %d, final %d; want 1, 1", codec.candidateCalls(value), codec.finalCalls(value)) } - if target == StageMerge && recorder.mergeSucceeded != 1 { - t.Fatalf("merge checkpoints = %d, want one", recorder.mergeSucceeded) - } - if target == StageNormalize && recorder.normalizeSucceeded != 1 { - t.Fatalf("normalize checkpoints = %d, want one", recorder.normalizeSucceeded) - } - checkpoint := recorder.mergeOutput - if target == StageNormalize { - checkpoint = recorder.normalizeOutput - } - var stored codecNotes - if err := json.Unmarshal(checkpoint.Artifact.Content, &stored); err != nil || !reflect.DeepEqual(stored, codecNotes{Items: []string{value}}) { - t.Fatalf("checkpoint content = %s, %v; want accepted value %q", checkpoint.Artifact.Content, err, value) - } - gotStages := make([]string, len(output.CheckpointEvents)) - for i, event := range output.CheckpointEvents { - gotStages[i] = event.Stage - } - wantStages := []string{"source", string(StageExtract), string(StageMerge), string(StageNormalize)} - if !reflect.DeepEqual(gotStages, wantStages) { - t.Fatalf("checkpoint event stages = %#v, want %#v", gotStages, wantStages) - } }) } } -func TestRunnerRecordsCandidateAndFinalEncodingFailures(t *testing.T) { - tests := []struct { - name string - target ModuleStage - value string - candidate bool - wantError string - }{ - {name: "merge candidate", target: StageMerge, value: "merge-candidate-failure", candidate: true, wantError: "serialize merge candidate"}, - {name: "normalize candidate", target: StageNormalize, value: "normalize-candidate-failure", candidate: true, wantError: "serialize normalize candidate"}, - {name: "merge final", target: StageMerge, value: "merge-final-failure", wantError: "serialize accepted merge output"}, - {name: "normalize final", target: StageNormalize, value: "normalize-final-failure", wantError: "serialize accepted normalize output"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - codec := &observedNotesCodec{} - if tc.candidate { - codec.candidateError = tc.value - } else { - codec.finalError = tc.value +func TestRunnerReportsCandidateAndFinalEncodingFailures(t *testing.T) { + for _, target := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { + for _, candidateFailure := range []bool{true, false} { + name := string(target) + "/final" + if candidateFailure { + name = string(target) + "/candidate" } - installObservedNotesCodec(t, prepared, codec) - configureCandidateOperation(prepared, tc.target, codecNotes{Items: []string{tc.value}}) - setCandidateValidator(prepared, tc.target, true) - recorder := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - debug := newCapturedDebugRecorder() - - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug}) - if err == nil || !strings.Contains(err.Error(), tc.wantError) { - t.Fatalf("Run() error = %v, want %q", err, tc.wantError) - } - attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", tc.target) - envelope := debug.envelope(t, attemptPath) - if !strings.Contains(envelope.Error, tc.wantError) { - t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError) - } - if tc.candidate && codec.finalCalls(tc.value) != 0 { - t.Fatalf("final encode calls = %d, want none after candidate failure", codec.finalCalls(tc.value)) - } - if tc.target == StageMerge && recorder.mergeSucceeded != 0 { - t.Fatalf("merge checkpoints = %d, want none", recorder.mergeSucceeded) - } - if tc.target == StageNormalize && recorder.normalizeSucceeded != 0 { - t.Fatalf("normalize checkpoints = %d, want none", recorder.normalizeSucceeded) - } - }) + t.Run(name, func(t *testing.T) { + prepared := preparedConcurrentPipeline(t, 1) + codec := &observedNotesCodec{} + value := "encoding-failure" + wantOperation := "encode" + if candidateFailure { + codec.candidateError = value + wantOperation = "encode candidate" + } else { + codec.finalError = value + } + installObservedNotesCodec(t, prepared, codec) + configureCandidateOperation(prepared, target, codecNotes{Items: []string{value}}) + setCandidateValidator(prepared, target, true) + output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) + var codecErr *ArtifactCodecOperationError + if !errors.As(err, &codecErr) || codecErr.Operation != wantOperation || codecErr.Kind != "test/notes" || len(output.OutputFiles) != 0 { + t.Fatalf("output=%#v error=%v codec error=%#v, want scoped %s failure without publication", output, err, codecErr, wantOperation) + } + if candidateFailure && codec.finalCalls(value) != 0 { + t.Fatal("candidate failure reached final encoding") + } + }) + } } } diff --git a/internal/framework/pipeline/runner_extract_correction_test.go b/internal/framework/pipeline/runner_extract_correction_test.go deleted file mode 100644 index 5d6e42ea..00000000 --- a/internal/framework/pipeline/runner_extract_correction_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package pipeline - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestRunnerCorrectsExtractCandidatesIndependentlyPerChunk(t *testing.T) { - prepared := preparedConcurrentPipeline(t, 2) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.Extract.Retries = 1 - - var mu sync.Mutex - corrections := make(map[int]*contracts.SemanticCorrection) - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - chunkIndex := request.Chunk.Index - if request.Correction != nil { - correction, err := contracts.CloneSemanticCorrection(request.Correction) - if err != nil { - return erasedTypedResult{}, err - } - mu.Lock() - corrections[chunkIndex] = correction - mu.Unlock() - candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("corrected-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1) - if err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("corrected-%d", chunkIndex)}}, ModelCandidate: candidate}, nil - } - candidate, err := contracts.NewModelCandidate([]byte(fmt.Sprintf("initial-response-%d", chunkIndex)), contracts.CorrectionProtocolSingleResponseV1) - if err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{fmt.Sprintf("invalid-%d", chunkIndex)}}, ModelCandidate: candidate}, nil - }) - lane.extractValidators.validators[0].typedValidate = func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) { - value := target.value.(codecNotes) - if value.Items[0] == fmt.Sprintf("invalid-%d", target.chunk.Index) { - return contracts.ValidationResult{Approved: false, ReasonCode: "incorrect_extract", Message: "candidate needs correction", CorrectionGuidance: fmt.Sprintf("correct chunk %d", target.chunk.Index)}, nil - } - return contracts.ValidationResult{Approved: true}, nil - } - - debug := newCapturedDebugRecorder() - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 2, Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 { - t.Fatalf("run output = %#v, want corrected accepted extracts", output) - } - mu.Lock() - defer mu.Unlock() - if len(corrections) != 2 { - t.Fatalf("corrections = %#v, want one correction per chunk", corrections) - } - for index := 0; index < 2; index++ { - correction := corrections[index] - if correction == nil || string(correction.AssistantResponse) != fmt.Sprintf("initial-response-%d", index) || !strings.Contains(correction.UserGuidance, fmt.Sprintf("correct chunk %d", index)) || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "incorrect_extract") || strings.Contains(correction.UserGuidance, "candidate needs correction") { - t.Fatalf("chunk %d correction = %#v, want its exact initial response and semantic replacement guidance only", index, correction) - } - } - terminal := string(debug.json["extract/notes/chunk-000001/terminal.json"]) - if !strings.Contains(terminal, `"kind":"semantic_correction"`) || !strings.Contains(terminal, `"producer_attempt_count":2`) { - t.Fatalf("extract terminal debug = %s, want correction provenance", terminal) - } - if strings.Contains(terminal, "initial-response-0") || strings.Contains(terminal, "corrected-response-0") || strings.Contains(terminal, "correct chunk 0") { - t.Fatalf("extract terminal debug leaked correction content: %s", terminal) - } - manifest, err := json.Marshal(output.Manifest) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(manifest), "initial-response-0") || strings.Contains(string(manifest), "corrected-response-0") || strings.Contains(string(manifest), "correct chunk 0") { - t.Fatalf("manifest leaked correction content: %s", manifest) - } -} - -func containsValidationSummary(summaries []artifacts.ValidationSummary, status string) bool { - for _, summary := range summaries { - if summary.Status == status { - return true - } - } - return false -} - -func TestRunnerContinuesValidationIncompleteExtractWithoutCheckpoint(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil - }) - lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, fmt.Errorf("validator unavailable") - } - recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - debug := newCapturedDebugRecorder() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 { - t.Fatalf("run output = %#v, want accepted incomplete extract", output) - } - if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") { - t.Fatalf("manifest validation = %#v, want incomplete extract provenance", output.Manifest) - } - if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" { - t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics) - } - if len(recorder.checkpoint.Outputs) != 0 || len(recorder.checkpoint.Rejected) != 0 { - t.Fatalf("extract checkpoint = %#v, want no persisted incomplete output", recorder.checkpoint) - } - encoded := string(debug.json["extract/notes/output.json"]) - if !strings.Contains(encoded, `"validation_incomplete_chunks":[0]`) { - t.Fatalf("extract output debug = %s, want incomplete chunk marker", encoded) - } -} diff --git a/internal/framework/pipeline/runner_extract_handoff_test.go b/internal/framework/pipeline/runner_extract_handoff_test.go index ae80b4af..fca46fae 100644 --- a/internal/framework/pipeline/runner_extract_handoff_test.go +++ b/internal/framework/pipeline/runner_extract_handoff_test.go @@ -3,8 +3,6 @@ package pipeline import ( "context" "encoding/json" - "fmt" - "reflect" "strings" "testing" @@ -12,11 +10,6 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -type extractCaptureRecorder struct { - CheckpointRecorder - checkpoint ExtractCheckpoint -} - type pipelineTypedMetadata map[string]any type mutatingMetadataValidator struct { @@ -35,48 +28,14 @@ func (v *mutatingMetadataValidator) Validate(_ context.Context, request contract return contracts.ValidationResult{Approved: true}, nil } -func (r *extractCaptureRecorder) ExtractSucceeded(_ string, _ string, _ []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error { - r.checkpoint = ExtractCheckpoint{ - Outputs: cloneCheckpointArtifacts(outputs), - Rejected: cloneRejectedOutputs(rejected), - } - return nil -} - -type extractResultLoader struct { - CheckpointLoader - checkpoint ExtractCheckpoint - decision CheckpointDecision -} - -func (l *extractResultLoader) Enabled() bool { return true } - -func (l *extractResultLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) { - return ExtractCheckpoint{ - Outputs: cloneCheckpointArtifacts(l.checkpoint.Outputs), - Rejected: cloneRejectedOutputs(l.checkpoint.Rejected), - }, l.decision -} - -func cloneCheckpointArtifacts(values []CheckpointArtifact) []CheckpointArtifact { - if len(values) == 0 { - return nil - } - cloned := make([]CheckpointArtifact, len(values)) - for i := range values { - cloned[i] = cloneCheckpointArtifact(values[i]) - } - return cloned -} - func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) + prepared := preparedConcurrentPipeline(t, 1) chunker := prepared.chunker.(*typedTestChunker) chunker.plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)} chunker.plan.Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)} var gotRange, gotPlan string - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { + installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { gotRange = string(request.Chunk.Annotations["same"]) gotPlan = string(request.Chunk.PlanAnnotations["same"]) request.Chunk.Annotations["same"][0] = '[' @@ -95,7 +54,7 @@ func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) { } func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) + prepared := preparedConcurrentPipeline(t, 1) input := prepared.input.(*typedTestInput) input.doc.Units[0].Metadata = map[string]any{ "typed": pipelineTypedMetadata{"raw": json.RawMessage(`{"value":true}`)}, @@ -114,7 +73,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes chunk: validator, }} var extractorSeen string - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { + installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { nested := request.Chunk.Units[0].Metadata["typed"].(pipelineTypedMetadata) extractorSeen = string(nested["raw"].(json.RawMessage)) nested["raw"].(json.RawMessage)[0] = '[' @@ -135,7 +94,7 @@ func TestRunnerPassesIndependentConcreteMetadataToValidatorsAndExtractors(t *tes } func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) + prepared := preparedConcurrentPipeline(t, 1) cyclic := make(map[string]any) cyclic["self"] = cyclic _, err := New().Run(context.Background(), RunInput{ @@ -147,144 +106,3 @@ func TestRunnerRejectsMetadataThatCannotBeCloned(t *testing.T) { t.Fatalf("Run() error = %v, want contextual metadata clone failure", err) } } - -func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - extractCalls := 0 - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - extractCalls++ - return erasedTypedResult{ - Value: typedValueForLane(0, request.Chunk.Index), - Diagnostics: []contracts.ProducerDiagnostic{{ - Disposition: contracts.DiagnosticDispositionObservation, - Category: contracts.DiagnosticCategoryNormalization, - ReasonCode: "accepted_extract_normalized", - OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{Scope: "extract", Message: "accepted extract"}}, - }}, - }, nil - }) - - freshDebug := newCapturedDebugRecorder() - recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - freshLoader := &extractResultLoader{ - CheckpointLoader: NoopCheckpointLoader(), - decision: CheckpointDecision{Reason: "extract checkpoint not found"}, - } - fresh, err := New().Run(context.Background(), RunInput{ - Prepared: prepared, - RawInput: []byte("input"), - Checkpoints: recorder, - Checkpoint: freshLoader, - Debug: freshDebug, - }) - if err != nil { - t.Fatalf("fresh Run() error = %v, want nil", err) - } - if extractCalls != 1 { - t.Fatalf("fresh extract calls = %d, want 1", extractCalls) - } - assertExtractDecision(t, fresh.CheckpointEvents, "executed", "extract checkpoint not found") - assertExtractDebugPaths(t, freshDebug, true) - - reusedDebug := newCapturedDebugRecorder() - reusedLoader := &extractResultLoader{ - CheckpointLoader: NoopCheckpointLoader(), - checkpoint: recorder.checkpoint, - decision: CheckpointDecision{Reused: true, Reason: "extract checkpoint matched"}, - } - reused, err := New().Run(context.Background(), RunInput{ - Prepared: prepared, - RawInput: []byte("input"), - Checkpoint: reusedLoader, - Debug: reusedDebug, - }) - if err != nil { - t.Fatalf("reused Run() error = %v, want nil", err) - } - if extractCalls != 1 { - t.Fatalf("extract calls after reuse = %d, want 1", extractCalls) - } - assertExtractDecision(t, reused.CheckpointEvents, "reused", "extract checkpoint matched") - assertExtractDebugPaths(t, reusedDebug, false) - if !reflect.DeepEqual(reused.NormalizeOutputs, fresh.NormalizeOutputs) { - t.Fatalf("reused normalize outputs = %#v, want fresh outputs %#v", reused.NormalizeOutputs, fresh.NormalizeOutputs) - } - if !reflect.DeepEqual(reused.Diagnostics, fresh.Diagnostics) { - t.Fatalf("reused diagnostics = %#v, want fresh diagnostics %#v", reused.Diagnostics, fresh.Diagnostics) - } -} - -func TestRunnerPromotesOnlyAcceptedExtractRetryDiagnostics(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - prepared.Steps[0].lanes[0].resolved.Extract.Retries = 1 - attempts := 0 - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - attempts++ - scope := "discarded" - if attempts == 2 { - scope = "accepted" - } - return erasedTypedResult{ - Value: typedValueForLane(0, request.Chunk.Index), - Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(scope, scope)}, - ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%s"]}`, scope)), - }, nil - }) - validatorCalls := 0 - prepared.Steps[0].lanes[0].extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - validatorCalls++ - return contracts.ValidationResult{Approved: validatorCalls == 2, ReasonCode: "retry", Message: "retry extract", CorrectionGuidance: "return an acceptable extract"}, nil - } - debug := newCapturedDebugRecorder() - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if attempts != 2 { - t.Fatalf("extract attempts = %d, want 2", attempts) - } - if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" { - t.Fatalf("promoted diagnostics = %#v, want accepted attempt only", output.Diagnostics) - } - assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1, 2) - first := debug.envelope(t, "extract/notes/chunk-000001/attempt-01.json") - if first.Error != "" || !strings.Contains(string(debug.json["extract/notes/chunk-000001/attempt-01.json"]), "rejection") { - t.Fatalf("first extract attempt = %#v, want rejection without error", first) - } - name := "extract/notes/chunk-000001/attempt-02.json" - if !debug.has(name) { - t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names()) - } -} - -func assertExtractDecision(t *testing.T, events []CheckpointEvent, action CheckpointDecisionCategory, reason string) { - t.Helper() - for _, event := range events { - if event.Stage == string(StageExtract) { - if event.Action != action || event.Reason != reason { - t.Fatalf("extract checkpoint event = %#v, want action %q reason %q", event, action, reason) - } - return - } - } - t.Fatalf("extract checkpoint event missing from %#v", events) -} - -func assertExtractDebugPaths(t *testing.T, debug *capturedDebugRecorder, wantAttempt bool) { - t.Helper() - for _, name := range []string{"extract/notes/input.json", "extract/notes/output.json"} { - if !debug.has(name) { - t.Fatalf("debug artifact %q is missing; names = %#v", name, debug.names()) - } - } - attemptPath := "extract/notes/chunk-000001/attempt-01.json" - if debug.has(attemptPath) != wantAttempt { - t.Fatalf("attempt debug path present = %t, want %t; names = %#v", debug.has(attemptPath), wantAttempt, debug.names()) - } - input := debug.envelope(t, "extract/notes/input.json") - wantReuse := !wantAttempt - if !strings.Contains(string(debug.json["extract/notes/input.json"]), `"reused":`+map[bool]string{true: "true", false: "false"}[wantReuse]) { - t.Fatalf("extract input payload = %#v, want reused %t", input.Payload, wantReuse) - } -} diff --git a/internal/framework/pipeline/runner_manifest_test.go b/internal/framework/pipeline/runner_manifest_test.go deleted file mode 100644 index 7c8e95cc..00000000 --- a/internal/framework/pipeline/runner_manifest_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package pipeline - -import ( - "reflect" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" -) - -func TestMergeLLMProfileManifestsDistinguishesEffectiveTargets(t *testing.T) { - got := mergeLLMProfileManifests( - []artifacts.LLMProfileManifest{ - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, - {ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "}, - }, - []artifacts.LLMProfileManifest{ - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, - }, - ) - want := []artifacts.LLMProfileManifest{ - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, - {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("merged profiles = %#v, want %#v", got, want) - } -} diff --git a/internal/framework/pipeline/runner_normalize_retry_test.go b/internal/framework/pipeline/runner_normalize_retry_test.go deleted file mode 100644 index 0a2cc613..00000000 --- a/internal/framework/pipeline/runner_normalize_retry_test.go +++ /dev/null @@ -1,380 +0,0 @@ -package pipeline - -import ( - "context" - "fmt" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestRunnerHandlesRetryableNormalizeFallbacks(t *testing.T) { - tests := []struct { - name string - retries int - operation func(int) erasedTypedResult - validator *preparedValidator - wantCalls int - wantItem string - wantMessages []string - wantRejected int - wantDebug []string - wantCheckpoint int - }{ - { - name: "accepts zero-retry fallback", - retries: 0, - operation: func(int) erasedTypedResult { - return retryableNormalizeResult("fallback", "ordinary", "fallback-warning") - }, - wantCalls: 1, - wantItem: "fallback", - wantMessages: []string{"ordinary", "fallback-warning"}, - wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`}, - wantCheckpoint: 1, - }, - { - name: "retries before accepting ordinary result", - retries: 1, - operation: func(attempt int) erasedTypedResult { - if attempt == 1 { - return retryableNormalizeResult("discarded", "discarded-ordinary", "discarded-fallback") - } - return erasedTypedResult{Value: codecNotes{Items: []string{"accepted"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted-warning")}} - }, - wantCalls: 2, - wantItem: "accepted", - wantMessages: []string{"accepted-warning"}, - wantDebug: []string{`"another_attempt":true`, `"fallback_accepted":false`}, - wantCheckpoint: 1, - }, - { - name: "accepts final fallback after exhaustion", - retries: 1, - operation: func(attempt int) erasedTypedResult { - return retryableNormalizeResult(fmt.Sprintf("fallback-%d", attempt), fmt.Sprintf("ordinary-%d", attempt), fmt.Sprintf("fallback-warning-%d", attempt)) - }, - wantCalls: 2, - wantItem: "fallback-2", - wantMessages: []string{"ordinary-2", "fallback-warning-2"}, - wantDebug: []string{`"another_attempt":false`, `"fallback_accepted":true`}, - wantCheckpoint: 1, - }, - { - name: "keeps final fallback rejection terminal", - retries: 1, - operation: func(int) erasedTypedResult { - return retryableNormalizeResult("rejected", "ordinary", "fallback-warning") - }, - validator: &preparedValidator{ - resolved: ResolvedValidator{Binding: Binding("reject-final-fallback"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "reject fallback", CorrectionGuidance: "return an acceptable normalized artifact"}, nil - }, - }, - wantCalls: 2, - wantRejected: 1, - wantDebug: []string{`"fallback_accepted":true`, `"rejection"`}, - wantCheckpoint: 0, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.Normalize.Retries = tc.retries - lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - if tc.validator != nil { - lane.normalizeValidators.validators = []preparedValidator{*tc.validator} - } - calls := 0 - lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - calls++ - return tc.operation(calls), nil - } - debug := newCapturedDebugRecorder() - checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug, Checkpoints: checkpoints}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if calls != tc.wantCalls { - t.Fatalf("normalize calls = %d, want %d", calls, tc.wantCalls) - } - if checkpoints.normalizeSucceeded != tc.wantCheckpoint { - t.Fatalf("normalize checkpoints = %d, want %d", checkpoints.normalizeSucceeded, tc.wantCheckpoint) - } - if len(output.Rejected) != tc.wantRejected { - t.Fatalf("rejected outputs = %#v, want %d", output.Rejected, tc.wantRejected) - } - var retryDebug strings.Builder - for _, name := range debug.names() { - if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") { - retryDebug.Write(debug.json[name]) - } - } - for _, fragment := range tc.wantDebug { - if !strings.Contains(retryDebug.String(), fragment) { - t.Fatalf("retry debug = %s, want %q", retryDebug.String(), fragment) - } - } - if tc.wantRejected != 0 { - if output.Rejected[0].AttemptCount != tc.wantCalls { - t.Fatalf("rejection attempt count = %d, want %d", output.Rejected[0].AttemptCount, tc.wantCalls) - } - return - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %#v, want one", output.NormalizeOutputs) - } - decoded, err := lane.typed.codec.decode(output.NormalizeOutputs[0].Artifact.Content) - if err != nil { - t.Fatalf("decode normalized output: %v", err) - } - normalized, ok := decoded.(codecNotes) - if !ok { - t.Fatalf("decoded normalized output = %T, want codecNotes", decoded) - } - if got := firstNote(normalized); got != tc.wantItem { - t.Fatalf("normalized item = %q, want %q", got, tc.wantItem) - } - var gotMessages []string - for _, group := range output.Diagnostics.Groups { - for _, sample := range group.Samples { - gotMessages = append(gotMessages, sample.Message) - } - } - if strings.Join(gotMessages, "|") != strings.Join(tc.wantMessages, "|") { - t.Fatalf("durable diagnostics = %#v, want %#v", gotMessages, tc.wantMessages) - } - }) - } -} - -func TestRunnerForwardsModuleRequestedNormalizeCorrection(t *testing.T) { - const ( - defective = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` - guidance = "Duplicate group 1 must use only supplied candidate numbers. Return one complete corrected response." - ) - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.Normalize.Retries = 1 - var observed *contracts.SemanticCorrection - calls := 0 - lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - calls++ - if request.Correction != nil { - clone, err := contracts.CloneSemanticCorrection(request.Correction) - if err != nil { - return erasedTypedResult{}, err - } - observed = clone - } - if calls == 1 { - return erasedTypedResult{ - Value: codecNotes{Items: []string{"safe fallback"}}, - ModelCandidate: attemptCandidate(t, defective), - Retry: &contracts.NormalizeRetry{ - ReasonCode: "semantic_proposal_invalid", - Message: "operator-facing proposal diagnostic", - CorrectionGuidance: guidance, - }, - }, nil - } - return erasedTypedResult{Value: codecNotes{Items: []string{"corrected"}}}, nil - } - debug := newCapturedDebugRecorder() - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if calls != 2 || observed == nil || string(observed.AssistantResponse) != defective || observed.UserGuidance != guidance { - t.Fatalf("normalize calls = %d correction = %#v", calls, observed) - } - if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 { - t.Fatalf("run output = %#v, want corrected accepted output", output) - } - var retryDebug strings.Builder - for _, name := range debug.names() { - if strings.HasPrefix(name, "normalize/notes/attempt-") && strings.HasSuffix(name, ".json") { - retryDebug.Write(debug.json[name]) - } - } - if !strings.Contains(retryDebug.String(), `"correction_available":true`) || strings.Contains(retryDebug.String(), guidance) || strings.Contains(retryDebug.String(), defective) { - t.Fatalf("retry debug = %s, want safe correction metadata without content", retryDebug.String()) - } -} - -func TestRunnerValidatesNormalizeRetryDiagnostics(t *testing.T) { - const ( - reasonSentinel = "reason-diagnostic-sentinel" - messageSentinel = "message-diagnostic-sentinel" - ) - reasonOverLimit := strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes-len(reasonSentinel)) + reasonSentinel + "x" - messageOverLimit := strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes-len(messageSentinel)) + messageSentinel + "x" - guidanceOverLimit := strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes-len(messageSentinel)) + messageSentinel + "x" - tests := []struct { - name string - retry contracts.NormalizeRetry - candidate *contracts.ModelCandidate - wantError string - hiddenValues []string - }{ - { - name: "accepts byte limits", - retry: contracts.NormalizeRetry{ - ReasonCode: strings.Repeat("r", contracts.MaxNormalizeRetryReasonCodeBytes), - Message: strings.Repeat("m", contracts.MaxNormalizeRetryMessageBytes), - }, - }, - { - name: "accepts correction guidance byte limit with candidate", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel, - CorrectionGuidance: strings.Repeat("g", contracts.MaxNormalizeRetryCorrectionGuidanceBytes), - }, - candidate: attemptCandidate(t, `{"duplicate_groups":[]}`), - }, - { - name: "rejects oversized reason code", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonOverLimit, - Message: messageSentinel, - }, - wantError: "reason code exceeds maximum length", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects oversized message", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageOverLimit, - }, - wantError: "message exceeds maximum length", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects invalid reason code UTF-8", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel + string([]byte{0xff}), - Message: messageSentinel, - }, - wantError: "reason code has invalid UTF-8", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects invalid message UTF-8", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel + string([]byte{0xff}), - }, - wantError: "message has invalid UTF-8", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects oversized correction guidance", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel, - CorrectionGuidance: guidanceOverLimit, - }, - wantError: "correction guidance exceeds maximum length", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects invalid correction guidance UTF-8", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel, - CorrectionGuidance: messageSentinel + string([]byte{0xff}), - }, - wantError: "correction guidance has invalid UTF-8", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects blank correction guidance", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel, - CorrectionGuidance: " \t\n ", - }, - wantError: "correction guidance is blank", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects correction guidance without candidate", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: messageSentinel, - CorrectionGuidance: "Return one complete corrected response.", - }, - wantError: "requires a model candidate", - hiddenValues: []string{reasonSentinel, messageSentinel}, - }, - { - name: "rejects blank reason code", - retry: contracts.NormalizeRetry{ - ReasonCode: " \t\n ", - Message: messageSentinel, - }, - wantError: "reason code is blank", - hiddenValues: []string{messageSentinel}, - }, - { - name: "rejects blank message", - retry: contracts.NormalizeRetry{ - ReasonCode: reasonSentinel, - Message: " \t\n ", - }, - wantError: "message is blank", - hiddenValues: []string{reasonSentinel}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - prepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - return erasedTypedResult{Value: codecNotes{Items: []string{"safe"}}, Retry: &tc.retry, ModelCandidate: tc.candidate}, nil - } - debug := newCapturedDebugRecorder() - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if tc.wantError == "" { - if err != nil { - t.Fatalf("Run() error = %v, want accepted retry fallback", err) - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %#v, want one accepted fallback", output.NormalizeOutputs) - } - return - } - if err == nil || !strings.Contains(err.Error(), tc.wantError) { - t.Fatalf("Run() error = %v, want fixed %q error", err, tc.wantError) - } - var debugOutput strings.Builder - for _, name := range debug.names() { - debugOutput.Write(debug.json[name]) - } - for _, value := range tc.hiddenValues { - if strings.Contains(err.Error(), value) || strings.Contains(debugOutput.String(), value) { - t.Fatalf("retry diagnostic leaked %q", value) - } - } - }) - } -} - -func retryableNormalizeResult(item, ordinary, fallback string) erasedTypedResult { - return erasedTypedResult{ - Value: codecNotes{Items: []string{item}}, - Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("attempt", ordinary)}, - Retry: &contracts.NormalizeRetry{ - ReasonCode: "retryable_normalization", - Message: "safe fallback is available", - FallbackDiagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("fallback", fallback)}, - }, - } -} diff --git a/internal/framework/pipeline/runner_rejection_warnings_test.go b/internal/framework/pipeline/runner_rejection_warnings_test.go deleted file mode 100644 index 47c62d57..00000000 --- a/internal/framework/pipeline/runner_rejection_warnings_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package pipeline - -import ( - "context" - "fmt" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type diagnosticChunker struct { - key string - plan source.ChunkPlan - calls int -} - -func (c *diagnosticChunker) Key() string { return c.key } -func (*diagnosticChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (c *diagnosticChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { - c.calls++ - return contracts.ChunkPlanResult{ - Plan: source.CloneChunkPlan(c.plan), - Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", c.calls), "operation diagnostic")}, - }, nil -} - -type chunkValidationFunc struct { - name string - validate func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) -} - -func (v chunkValidationFunc) Name() string { return v.name } -func (chunkValidationFunc) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassDeterministic -} -func (v chunkValidationFunc) Validate(ctx context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return v.validate(ctx, request) -} - -func TestRunnerPromotesOnlyTerminalRejectionDiagnostics(t *testing.T) { - for _, target := range []ModuleStage{StageChunk, StageExtract, StageMerge, StageNormalize} { - t.Run(string(target), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - attempts := 0 - first := func() contracts.ValidationResult { - return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", attempts), "validator diagnostic")}} - } - reject := func() contracts.ValidationResult { - return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected", CorrectionGuidance: "return an acceptable candidate"} - } - debug := newCapturedDebugRecorder() - recorder := &extractCaptureRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - - switch target { - case StageChunk: - prepared.resolved.ChunkValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - chunker := prepared.chunker.(*typedTestChunker) - prepared.chunker = &diagnosticChunker{key: prepared.resolved.Chunk.Module, plan: source.CloneChunkPlan(chunker.plan)} - prepared.resolved.Chunk.Retries = 1 - prepared.chunkValidators.validators = []preparedValidator{ - {resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-approval", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return first(), nil - }}}, - {resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetChunk}, chunk: chunkValidationFunc{name: "warning-rejection", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return reject(), nil - }}}, - } - chunkerWithDiagnostics := prepared.chunker.(*diagnosticChunker) - first = func() contracts.ValidationResult { - return contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("validator-%d", chunkerWithDiagnostics.calls), "validator diagnostic")}} - } - case StageExtract: - lane.resolved.Extract.Retries = 1 - lane.resolved.ExtractValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) { - attempts++ - return erasedTypedResult{Value: codecNotes{Items: []string{"extract"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil - }) - lane.extractValidators.validators = rejectionDiagnosticTypedValidators(first, reject) - case StageMerge: - lane.resolved.Merge.Retries = 1 - lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - attempts++ - return erasedTypedResult{Value: codecNotes{Items: []string{"merge"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil - } - lane.mergeValidators.validators = rejectionDiagnosticTypedValidators(first, reject) - case StageNormalize: - lane.resolved.Normalize.Retries = 1 - lane.resolved.NormalizeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - attempts++ - return erasedTypedResult{Value: codecNotes{Items: []string{"normalize"}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic(fmt.Sprintf("operation-%d", attempts), "operation diagnostic")}, ModelCandidate: attemptCandidate(t, fmt.Sprintf(`{"items":["%d"]}`, attempts))}, nil - } - lane.normalizeValidators.validators = rejectionDiagnosticTypedValidators(first, reject) - } - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: recorder, Debug: debug}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - wantScopes := []string{"operation-2", "validator-2"} - wantAttempts := 2 - if target == StageChunk { - wantScopes = []string{"operation-1", "validator-1"} - wantAttempts = 1 - } - if got := rejectionDiagnosticScopes(output.Diagnostics.Groups); !reflect.DeepEqual(got, wantScopes) { - t.Fatalf("published diagnostic scopes = %#v, want %#v", got, wantScopes) - } - if len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != wantAttempts { - t.Fatalf("rejections = %#v, want terminal rejection after %d attempt(s)", output.Rejected, wantAttempts) - } - attemptPath := fmt.Sprintf("%s/notes/attempt-01.json", target) - if target == StageChunk { - attemptPath = "chunk/attempt-01.json" - } else if target == StageExtract { - attemptPath = "extract/notes/chunk-000001/attempt-01.json" - } - if !strings.Contains(string(debug.json[attemptPath]), "rejection") { - t.Fatalf("first attempt debug = %s, want rejection", debug.json[attemptPath]) - } - }) - } -} - -func rejectionDiagnosticTypedValidators(first func() contracts.ValidationResult, reject func() contracts.ValidationResult) []preparedValidator { - return []preparedValidator{ - {resolved: ResolvedValidator{Binding: Binding("warning-approval"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return first(), nil - }}, - {resolved: ResolvedValidator{Binding: Binding("warning-rejection"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return reject(), nil - }}, - } -} - -func rejectionDiagnosticScopes(groups []contracts.DiagnosticGroup) []string { - scopes := make([]string, len(groups)) - for index := range groups { - scopes[index] = groups[index].Samples[0].Scope - } - return scopes -} diff --git a/internal/framework/pipeline/runner_reuse_lineage_test.go b/internal/framework/pipeline/runner_reuse_lineage_test.go deleted file mode 100644 index e08d7041..00000000 --- a/internal/framework/pipeline/runner_reuse_lineage_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package pipeline - -import ( - "context" - "fmt" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type reuseLineageCheckpointSpy struct { - CheckpointLoader - CheckpointRecorder - loads map[string]int - writes map[string]int - forbid map[string]struct{} -} - -func newReuseLineageCheckpointSpy() *reuseLineageCheckpointSpy { - return &reuseLineageCheckpointSpy{ - CheckpointLoader: NoopCheckpointLoader(), - CheckpointRecorder: NoopCheckpointRecorder(), - loads: make(map[string]int), - writes: make(map[string]int), - forbid: make(map[string]struct{}), - } -} - -func (s *reuseLineageCheckpointSpy) Enabled() bool { return true } - -func (s *reuseLineageCheckpointSpy) load(stage, laneID string) { - key := stage + "/" + laneID - if _, forbidden := s.forbid[key]; forbidden { - panic("checkpoint lookup crossed validation-incomplete lineage: " + key) - } - s.loads[key]++ -} - -func (s *reuseLineageCheckpointSpy) write(stage, action, laneID string) { - s.writes[stage+"/"+action+"/"+laneID]++ -} - -func (s *reuseLineageCheckpointSpy) Extract(laneID, _ string, _ []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) { - s.load("extract", laneID) - return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing) -} - -func (s *reuseLineageCheckpointSpy) Merge(laneID, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) { - s.load("merge", laneID) - return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing) -} - -func (s *reuseLineageCheckpointSpy) Normalize(laneID, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) { - s.load("normalize", laneID) - return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing) -} - -func (s *reuseLineageCheckpointSpy) ExtractRunning(laneID, _ string, _ []CheckpointFingerprint) error { - s.write("extract", "running", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) ExtractSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ []CheckpointArtifact, _ []contracts.RejectedOutput) error { - s.write("extract", "succeeded", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) ExtractFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error { - s.write("extract", "failed", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) MergeRunning(laneID, _ string, _ []CheckpointFingerprint) error { - s.write("merge", "running", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) MergeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error { - s.write("merge", "succeeded", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) MergeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error { - s.write("merge", "rejected", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) MergeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error { - s.write("merge", "failed", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) NormalizeRunning(laneID, _ string, _ []CheckpointFingerprint) error { - s.write("normalize", "running", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) NormalizeSucceeded(laneID, _ string, _ []CheckpointFingerprint, _ CheckpointArtifact) error { - s.write("normalize", "succeeded", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) NormalizeRejected(laneID, _ string, _ []CheckpointFingerprint, _ contracts.RejectedOutput) error { - s.write("normalize", "rejected", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) NormalizeFailed(laneID, _ string, _ []CheckpointFingerprint, _ error) error { - s.write("normalize", "failed", laneID) - return nil -} - -func (s *reuseLineageCheckpointSpy) writesFor(stage, laneID string) int { - total := 0 - needle := stage + "/" - suffix := "/" + laneID - for key, count := range s.writes { - if strings.HasPrefix(key, needle) && strings.HasSuffix(key, suffix) { - total += count - } - } - return total -} - -func unavailableTypedValidator(kind contracts.ArtifactKind) preparedValidator { - return preparedValidator{ - resolved: ResolvedValidator{Binding: Binding("typed/check"), Target: ValidatorTargetTyped, ArtifactKind: kind}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, fmt.Errorf("validator unavailable") - }, - } -} - -func TestValidationIncompleteExtractDisablesDownstreamCheckpointIO(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.ExtractValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil - }) - lane.extractValidators.validators[0].typedValidate = func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, fmt.Errorf("validator unavailable") - } - spy := newReuseLineageCheckpointSpy() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs)) - } - if spy.loads["merge/notes"] != 0 || spy.loads["normalize/notes"] != 0 { - t.Fatalf("downstream loads = %#v, want none", spy.loads) - } - if spy.writesFor("merge", "notes") != 0 || spy.writesFor("normalize", "notes") != 0 { - t.Fatalf("downstream writes = %#v, want none", spy.writes) - } - if spy.writes["extract/succeeded/notes"] != 0 { - t.Fatalf("extract writes = %#v, want no reusable success", spy.writes) - } -} - -func TestValidationIncompleteMergeDisablesNormalizeCheckpointIO(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.MergeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil - }) - lane.mergeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)} - spy := newReuseLineageCheckpointSpy() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalize outputs = %d, want current-run handoff", len(output.NormalizeOutputs)) - } - if spy.loads["merge/notes"] != 1 || spy.loads["normalize/notes"] != 0 { - t.Fatalf("loads = %#v, want merge lookup only", spy.loads) - } - if spy.writes["merge/succeeded/notes"] != 0 || spy.writesFor("normalize", "notes") != 0 { - t.Fatalf("writes = %#v, want no reusable merge or normalize state", spy.writes) - } -} - -func TestValidationIncompleteNormalizeIsNotPublished(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil - }) - lane.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(lane.resolved.ArtifactKind)} - spy := newReuseLineageCheckpointSpy() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if len(output.NormalizeOutputs) != 1 || spy.writes["normalize/succeeded/notes"] != 0 { - t.Fatalf("output/writes = %d / %#v, want in-memory output without normalize publication", len(output.NormalizeOutputs), spy.writes) - } -} - -func TestGeneratedReferenceFromIncompleteValidationDisablesDependentCheckpointIO(t *testing.T) { - input, _, _ := handoffFixture(t, codecNotes{Items: []string{"producer"}}) - prepared := input.Prepared - producer := &prepared.Steps[0].lanes[0] - consumer := &prepared.Steps[1].lanes[0] - producer.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - producer.normalizeValidators.validators = []preparedValidator{unavailableTypedValidator(producer.resolved.ArtifactKind)} - referenceItems := 0 - consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - referenceItems = len(request.References.Slots["producer-output"].Items) - return erasedTypedResult{Value: codecScore{Value: 1}}, nil - } - spy := newReuseLineageCheckpointSpy() - for _, stage := range []string{"extract", "merge", "normalize"} { - spy.forbid[stage+"/score"] = struct{}{} - } - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: spy, Checkpoints: spy}) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if referenceItems != 1 || len(output.NormalizeOutputs) != 2 { - t.Fatalf("current-run handoff = items %d outputs %d, want one generated item and two outputs", referenceItems, len(output.NormalizeOutputs)) - } - for _, stage := range []string{"extract", "merge", "normalize"} { - if spy.writesFor(stage, "score") != 0 { - t.Fatalf("dependent writes = %#v, want none for %s", spy.writes, stage) - } - } -} diff --git a/internal/framework/pipeline/runner_semantic_correction_test.go b/internal/framework/pipeline/runner_semantic_correction_test.go deleted file mode 100644 index dec4628d..00000000 --- a/internal/framework/pipeline/runner_semantic_correction_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package pipeline - -import ( - "context" - "fmt" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestRunnerCorrectsRejectedMergeAndNormalizeCandidates(t *testing.T) { - for _, target := range []ModuleStage{StageMerge, StageNormalize} { - t.Run(string(target), func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - var correction *contracts.SemanticCorrection - validator := preparedValidator{ - resolved: ResolvedValidator{Binding: Binding("correction-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(_ context.Context, _ any, target typedValidationTarget) (contracts.ValidationResult, error) { - value := target.value.(codecNotes) - if firstNote(value) == "invalid" { - return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "candidate needs correction", CorrectionGuidance: "produce the accepted value"}, nil - } - return contracts.ValidationResult{Approved: true}, nil - }, - } - operation := func(observed *contracts.SemanticCorrection) (erasedTypedResult, error) { - if observed != nil { - cloned, err := contracts.CloneSemanticCorrection(observed) - if err != nil { - return erasedTypedResult{}, err - } - correction = cloned - } - item, response := "invalid", `{"items":["invalid"]}` - if observed != nil { - item, response = "accepted", `{"items":["accepted"]}` - } - candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1) - if err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: codecNotes{Items: []string{item}}, ModelCandidate: candidate}, nil - } - switch target { - case StageMerge: - lane.resolved.Merge.Retries = 1 - lane.mergeValidators.validators = []preparedValidator{validator} - lane.typed.merge = func(_ context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - return operation(request.Correction) - } - case StageNormalize: - lane.resolved.Normalize.Retries = 1 - lane.normalizeValidators.validators = []preparedValidator{validator} - lane.typed.normalize = func(_ context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { - return operation(request.Correction) - } - } - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if correction == nil || string(correction.AssistantResponse) != `{"items":["invalid"]}` || !strings.Contains(correction.UserGuidance, "produce the accepted value") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "invalid_candidate") || strings.Contains(correction.UserGuidance, "candidate needs correction") { - t.Fatalf("%s correction = %#v, want exact rejected response and semantic replacement guidance only", target, correction) - } - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { - t.Fatalf("run output = %#v, want corrected accepted output", output) - } - }) - } -} - -func TestRunnerRejectsDeterministicMergeCandidateWithoutCorrection(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.Merge.Retries = 1 - lane.resolved.MergeValidationPolicy.SemanticRejection = SemanticRejectionRejectOutput - calls := 0 - lane.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { - calls++ - return erasedTypedResult{Value: codecNotes{Items: []string{"invalid"}}}, nil - } - lane.mergeValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("reject"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: false, ReasonCode: "invalid_candidate", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil - }, - }} - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}) - if err != nil { - t.Fatalf("Run() error = %v, want non-fatal rejection", err) - } - if calls != 1 || len(output.Rejected) != 1 || output.Rejected[0].AttemptCount != 1 { - t.Fatalf("merge calls = %d output = %#v, want one deterministic terminal rejection", calls, output) - } -} - -func TestRunnerContinuesNormalizeAfterValidatorFailure(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - lane := &prepared.Steps[0].lanes[0] - lane.resolved.NormalizeValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue - lane.normalizeValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("unavailable"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, fmt.Errorf("validator unavailable") - }, - }} - checkpoints := &candidateCheckpointRecorder{CheckpointRecorder: NoopCheckpointRecorder()} - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoints: checkpoints}) - if err != nil { - t.Fatalf("Run() error = %v, want incomplete accepted output", err) - } - if len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 0 || checkpoints.normalizeSucceeded != 0 { - t.Fatalf("run output = %#v checkpoints = %#v, want non-checkpointed incomplete normalized output", output, checkpoints) - } - if output.Manifest.ValidationStatus != "incomplete" || !containsValidationSummary(output.Manifest.ValidationSummaries, "incomplete") { - t.Fatalf("manifest validation = %#v, want incomplete normalization provenance", output.Manifest) - } - if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" { - t.Fatalf("diagnostics = %#v, want validation-incomplete group", output.Diagnostics) - } -} diff --git a/internal/framework/pipeline/runner_session_test.go b/internal/framework/pipeline/runner_session_test.go index 9ccff42b..96b7cd58 100644 --- a/internal/framework/pipeline/runner_session_test.go +++ b/internal/framework/pipeline/runner_session_test.go @@ -3,34 +3,39 @@ package pipeline import ( "context" "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestRunnerUsesProvidedSessionWithoutSourceFallback(t *testing.T) { for _, test := range []struct { name string session string - wantSet bool want string }{ - {name: "provided", session: " supplied-session ", wantSet: true, want: "supplied-session"}, - {name: "empty", wantSet: false}, + {name: "provided", session: " supplied-session ", want: "supplied-session"}, + {name: "empty"}, } { t.Run(test.name, func(t *testing.T) { - prepared, _ := preparedTerminalDebugPipeline(t) - output, err := New().Run(context.Background(), RunInput{ + prepared := preparedConcurrentPipeline(t, 1) + var session string + calls := 0 + installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { + calls++ + session = request.SessionID + return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil + }) + _, err := New().Run(context.Background(), RunInput{ Prepared: prepared, RawInput: []byte("input"), SessionID: test.session, + Workers: 1, }) if err != nil { t.Fatal(err) } - value, found := output.Manifest.Metadata["session_id"] - if found != test.wantSet { - t.Fatalf("manifest session presence = %t, want %t; metadata = %#v", found, test.wantSet, output.Manifest.Metadata) - } - if found && value != test.want { - t.Fatalf("manifest session = %#v, want %q", value, test.want) + if calls != 1 || session != test.want { + t.Fatalf("extract calls=%d session=%q, want one call and %q", calls, session, test.want) } }) } diff --git a/internal/framework/pipeline/runner_terminal_debug_test.go b/internal/framework/pipeline/runner_terminal_debug_test.go deleted file mode 100644 index 44882392..00000000 --- a/internal/framework/pipeline/runner_terminal_debug_test.go +++ /dev/null @@ -1,472 +0,0 @@ -package pipeline - -import ( - "context" - "errors" - "fmt" - "reflect" - "sort" - "strconv" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -type terminalChunker struct { - key string - plan source.ChunkPlan - diagnostics []contracts.ProducerDiagnostic - err error - calls *int -} - -func (c terminalChunker) Key() string { return c.key } - -func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } - -func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { - if c.calls != nil { - (*c.calls)++ - } - return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: contracts.CloneProducerDiagnostics(c.diagnostics)}, c.err -} - -type terminalChunkValidator struct { - result contracts.ValidationResult - err error -} - -type retryingLLMChunkValidator struct { - requests []contracts.ChunkValidationRequest - calls int -} - -type observingChunkValidator struct { - request contracts.ChunkValidationRequest -} - -type observingSerializedValidator struct { - request contracts.SerializedValidationRequest -} - -func (*observingChunkValidator) Name() string { return "observing/chunk-validator" } -func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassDeterministic -} -func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - v.request = request - return contracts.ValidationResult{Approved: true}, nil -} - -func (*observingSerializedValidator) Name() string { return "observing/serialized-validator" } -func (*observingSerializedValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassDeterministic -} -func (v *observingSerializedValidator) Validate(_ context.Context, request contracts.SerializedValidationRequest) (contracts.ValidationResult, error) { - v.request = request - return contracts.ValidationResult{Approved: true}, nil -} - -func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" } - -func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassDeterministic -} - -func (v terminalChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return v.result, v.err -} - -func (*retryingLLMChunkValidator) Name() string { return "retrying/llm-chunk-validator" } - -func (*retryingLLMChunkValidator) ExecutionClass() contracts.ExecutionClass { - return contracts.ExecutionClassLLMBacked -} - -func (validator *retryingLLMChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - validator.calls++ - validator.requests = append(validator.requests, request) - if validator.calls == 1 && len(request.Chunks) > 0 { - request.Chunks[0].ID = "mutated" - } - if validator.calls == 1 { - return contracts.ValidationResult{}, errors.New("temporary validator failure") - } - return contracts.ValidationResult{Approved: true}, nil -} - -func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, prefix string, attempts ...int) { - t.Helper() - marker := strings.TrimSuffix(prefix, "/") + "/attempt-" - var got []int - for _, name := range debug.names() { - if !strings.HasPrefix(name, marker) || !strings.HasSuffix(name, ".json") { - continue - } - remainder := strings.TrimSuffix(strings.TrimPrefix(name, marker), ".json") - if strings.Contains(remainder, "/") { - continue - } - value, err := strconv.Atoi(remainder) - if err != nil { - t.Fatalf("parse attempt index from %q: %v", name, err) - } - got = append(got, value) - } - sort.Ints(got) - if !reflect.DeepEqual(got, attempts) { - t.Fatalf("attempt envelopes under %q = %#v, want %#v; names = %#v", prefix, got, attempts, debug.names()) - } -} - -func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) { - t.Helper() - prepared := preparedAttemptDebugPipeline(t) - prepared.resolved.ChunkValidationPolicy = ValidationPolicy{ - ProducerStructuralFailure: ProducerStructuralFailureFailRun, - SemanticRejection: SemanticRejectionRejectOutput, - ValidatorFailure: ValidatorFailureFailRun, - } - prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{ - ProducerStructuralFailure: ProducerStructuralFailureFailRun, - SemanticRejection: SemanticRejectionRejectOutput, - ValidatorFailure: ValidatorFailureFailRun, - } - prepared.Steps[0].lanes[0].resolved.MergeValidationPolicy = ValidationPolicy{ - ProducerStructuralFailure: ProducerStructuralFailureFailRun, - SemanticRejection: SemanticRejectionRejectOutput, - ValidatorFailure: ValidatorFailureFailRun, - } - prepared.Steps[0].lanes[0].resolved.NormalizeValidationPolicy = ValidationPolicy{ - ProducerStructuralFailure: ProducerStructuralFailureFailRun, - SemanticRejection: SemanticRejectionRejectOutput, - ValidatorFailure: ValidatorFailureFailRun, - } - chunker, ok := prepared.chunker.(*typedTestChunker) - if !ok { - t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker) - } - return prepared, source.CloneChunkPlan(chunker.plan) -} - -func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) { - tests := []struct { - name string - moduleError error - validator terminalChunkValidator - wantError string - wantRejection bool - attemptError bool - }{ - {name: "accepted", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}}, - {name: "module error", moduleError: errors.New("chunk module failed"), wantError: "chunk module failed", attemptError: true}, - {name: "validator rejection", validator: terminalChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "chunk_rejected", Message: "chunk rejected", CorrectionGuidance: "return an acceptable chunk plan"}}, wantRejection: true}, - {name: "validator error", validator: terminalChunkValidator{err: errors.New("chunk validator failed")}, wantError: "chunk validator failed"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("chunk", "chunk diagnostic")}, err: tc.moduleError} - prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}} - debug := newCapturedDebugRecorder() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - assertAttemptEnvelopeSequence(t, debug, "chunk", 1) - envelope := debug.envelope(t, "chunk/attempt-01.json") - if tc.wantError != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantError) { - t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError) - } - if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) { - t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError) - } - if !tc.attemptError && envelope.Error != "" { - t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error) - } - } else if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if tc.wantRejection { - if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "chunk_rejected" { - t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected) - } - } - if tc.name == "accepted" { - encoded := string(debug.json["chunk/attempt-01.json"]) - if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) { - t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded) - } - } - }) - } -} - -func TestRunnerRetriesLLMValidatorsWithoutRegeneratingChunkCandidate(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - chunkCalls := 0 - prepared.resolved.Chunk.Retries = 0 - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &chunkCalls} - validator := &retryingLLMChunkValidator{} - prepared.chunkValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: ModuleBinding{Module: validator.Name(), Retries: 1}, ExecutionClass: contracts.ExecutionClassLLMBacked, Target: ValidatorTargetChunk}, - chunk: validator, - }} - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil { - t.Fatalf("Run() error = %v", err) - } - if chunkCalls != 1 || validator.calls != 2 { - t.Fatalf("producer calls = %d validator calls = %d, want 1 and 2", chunkCalls, validator.calls) - } - if len(validator.requests) != 2 || len(validator.requests[0].Chunks) == 0 || len(validator.requests[1].Chunks) == 0 || validator.requests[1].Chunks[0].ID == "mutated" { - t.Fatalf("validator requests = %#v, want independently owned immutable candidates", validator.requests) - } -} - -func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)} - plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)} - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan} - validator := &observingChunkValidator{} - prepared.chunkValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, - chunk: validator, - }} - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if len(validator.request.Chunks) != 1 { - t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks) - } - chunk := validator.request.Chunks[0] - if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" { - t.Fatalf("materialized chunk identity = %#v", chunk) - } - if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` { - t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"]) - } - if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 { - t.Fatalf("materialized metadata = %#v", chunk.Metadata) - } -} - -func TestRunnerForwardsDetachedRepairAttemptsToChunkValidators(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan} - chunkValidator := &observingChunkValidator{} - serializedValidator := &observingSerializedValidator{} - chunkAttempts := 2 - serializedAttempts := 3 - chunkBinding := Binding(chunkValidator.Name()) - chunkBinding.StructuredOutputRepairAttempts = &chunkAttempts - serializedBinding := Binding(serializedValidator.Name()) - serializedBinding.StructuredOutputRepairAttempts = &serializedAttempts - prepared.chunkValidators.validators = []preparedValidator{ - {resolved: ResolvedValidator{Binding: chunkBinding, Target: ValidatorTargetChunk}, chunk: chunkValidator}, - {resolved: ResolvedValidator{Binding: serializedBinding, Target: ValidatorTargetSerialized}, serialized: serializedValidator}, - } - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if got := chunkValidator.request.StructuredOutputRepairAttempts; got == nil || *got != chunkAttempts || got == chunkBinding.StructuredOutputRepairAttempts { - t.Fatalf("chunk validator repair attempts = %v, want detached value %d", got, chunkAttempts) - } - if got := serializedValidator.request.StructuredOutputRepairAttempts; got == nil || *got != serializedAttempts || got == serializedBinding.StructuredOutputRepairAttempts { - t.Fatalf("serialized validator repair attempts = %v, want detached value %d", got, serializedAttempts) - } -} - -func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)} - prepared.resolved.Chunk.Retries = 1 - calls := 0 - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls} - debug := newCapturedDebugRecorder() - - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err == nil || !strings.Contains(err.Error(), "valid JSON") { - t.Fatalf("Run() error = %v, want malformed annotation error", err) - } - if calls != 2 { - t.Fatalf("plan calls = %d, want two attempts", calls) - } - assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2) -} - -func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) { - tests := []struct { - name string - moduleError error - validatorErr error - reject bool - candidateFail bool - finalFail bool - wantError string - attemptError bool - }{ - {name: "terminal rejection", reject: true}, - {name: "module error", moduleError: errors.New("extract module failed"), wantError: "extract module failed", attemptError: true}, - {name: "validator error", validatorErr: errors.New("extract validator failed"), wantError: "extract validator failed"}, - {name: "candidate codec error", candidateFail: true, wantError: "serialize extract candidate", attemptError: true}, - {name: "final codec error", finalFail: true, wantError: "serialize accepted extract output", attemptError: true}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - prepared.Steps[0].lanes[0].resolved.ExtractValidationPolicy = ValidationPolicy{ - ProducerStructuralFailure: ProducerStructuralFailureFailRun, - SemanticRejection: SemanticRejectionRejectOutput, - ValidatorFailure: ValidatorFailureFailRun, - } - value := "extract-terminal" - codec := &observedNotesCodec{} - if tc.candidateFail { - codec.candidateError = value - } - if tc.finalFail { - codec.finalError = value - } - installObservedNotesCodec(t, prepared, codec) - installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) { - if tc.moduleError != nil { - return erasedTypedResult{}, tc.moduleError - } - return erasedTypedResult{Value: codecNotes{Items: []string{value}}, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("extract", "extract diagnostic")}}, nil - }) - prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("terminal/extract-validator"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: !tc.reject, ReasonCode: "extract_rejected", Message: "extract rejected", CorrectionGuidance: "return an acceptable extract"}, tc.validatorErr - }, - }} - debug := newCapturedDebugRecorder() - - output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - assertAttemptEnvelopeSequence(t, debug, "extract/notes/chunk-000001", 1) - attemptPath := "extract/notes/chunk-000001/attempt-01.json" - envelope := debug.envelope(t, attemptPath) - if tc.wantError != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantError) { - t.Fatalf("Run() error = %v, attempt error = %q; want %q", err, envelope.Error, tc.wantError) - } - if tc.attemptError && !strings.Contains(envelope.Error, tc.wantError) { - t.Fatalf("attempt error = %q, want %q", envelope.Error, tc.wantError) - } - if !tc.attemptError && envelope.Error != "" { - t.Fatalf("settled validator failure attempt error = %q, want empty", envelope.Error) - } - } else { - if err != nil { - t.Fatalf("Run() error = %v, want nil rejection", err) - } - if envelope.Error != "" || len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "extract_rejected" { - t.Fatalf("extract rejection = envelope %#v, outputs %#v", envelope, output.Rejected) - } - } - }) - } -} - -func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) { - t.Run("chunk", func(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - primary := errors.New("chunk operation failed") - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary} - debug := newCapturedDebugRecorder() - debug.failPath = "chunk/attempt-01.json" - - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { - t.Fatalf("Run() error = %v, want joined operation and debug errors", err) - } - }) - - t.Run("extract", func(t *testing.T) { - prepared := preparedAttemptDebugPipeline(t) - primary := errors.New("extract operation failed") - installExtractOperation(prepared, 0, func(context.Context, contracts.TypedExtractionRequest) (erasedTypedResult, error) { - return erasedTypedResult{}, primary - }) - debug := newCapturedDebugRecorder() - debug.failPath = "extract/notes/chunk-000001/attempt-01.json" - - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err == nil || !errors.Is(err, primary) || !strings.Contains(err.Error(), "write extract attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { - t.Fatalf("Run() error = %v, want joined operation and debug errors", err) - } - }) -} - -func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) { - prepared, plan := preparedTerminalDebugPipeline(t) - prepared.resolved.Chunk.Retries = 1 - calls := 0 - prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls} - prepared.chunkValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, - chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}}, - }} - debug := newCapturedDebugRecorder() - debug.failPath = "chunk/attempt-01.json" - - _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}) - if err == nil || !strings.Contains(err.Error(), "write chunk attempt debug artifact") || !strings.Contains(err.Error(), "debug recorder failure") { - t.Fatalf("Run() error = %v, want terminal attempt debug failure", err) - } - if calls != 1 { - t.Fatalf("chunk calls = %d, want one attempt without retry", calls) - } - if debug.has("chunk/attempt-02.json") { - t.Fatal("second chunk attempt envelope exists after non-retryable debug persistence failure") - } -} - -func TestRunnerKeepsExtractModuleAndValidatorLLMCallsIsolated(t *testing.T) { - prepared := preparedAttemptDebugPipelineWithChunks(t, 2) - debug := newCapturedDebugRecorder() - client := WithDebugLLMRecording(attemptDebugLLM{}, debug) - installExtractOperation(prepared, 0, func(ctx context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) { - if err := callAttemptDebugLLM(ctx, client, "extract-module"); err != nil { - return erasedTypedResult{}, err - } - return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil - }) - prepared.Steps[0].lanes[0].extractValidators.validators = []preparedValidator{{ - resolved: ResolvedValidator{Binding: Binding("llm-check"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}, - typedValidate: func(ctx context.Context, _ any, _ typedValidationTarget) (contracts.ValidationResult, error) { - if err := callAttemptDebugLLM(ctx, client, "extract-validator"); err != nil { - return contracts.ValidationResult{}, err - } - return contracts.ValidationResult{Approved: true}, nil - }, - }} - - if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug}); err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - validatorResponses := make(map[string]struct{}) - for chunkNumber := 1; chunkNumber <= 2; chunkNumber++ { - chunkPath := fmt.Sprintf("chunk-%06d", chunkNumber) - module := debug.envelope(t, "extract/notes/"+chunkPath+"/attempt-01.json") - validator := debug.envelope(t, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/01-llm-check-attempt-01.json") - if len(module.LLMCalls) != 1 || !strings.Contains(module.LLMCalls[0].ResponsePath, "extract/notes/"+chunkPath+"/attempt-01/") { - t.Fatalf("module LLM calls for %s = %#v, want extract module call only", chunkPath, module.LLMCalls) - } - if len(validator.LLMCalls) != 1 || !strings.Contains(validator.LLMCalls[0].ResponsePath, "validate/extract/notes/typed%2Fextract-notes/"+chunkPath+"/") { - t.Fatalf("validator LLM calls for %s = %#v, want chunk-scoped validator call only", chunkPath, validator.LLMCalls) - } - if module.LLMCalls[0].CallID == validator.LLMCalls[0].CallID { - t.Fatalf("module and validator attempts for %s share LLM call %#v", chunkPath, module.LLMCalls) - } - validatorResponses[validator.LLMCalls[0].ResponsePath] = struct{}{} - } - if len(validatorResponses) != 2 { - t.Fatalf("validator response paths = %#v, want one distinct path per chunk", validatorResponses) - } -} diff --git a/internal/framework/pipeline/validation_policy_test.go b/internal/framework/pipeline/validation_policy_test.go index 78547d79..3ee424fb 100644 --- a/internal/framework/pipeline/validation_policy_test.go +++ b/internal/framework/pipeline/validation_policy_test.go @@ -18,10 +18,10 @@ func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) { invocation *contracts.CacheMode want contracts.ProducerControls }{ - {"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheAuto}}, - {"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModePermissive, 5, contracts.CacheRefresh}}, - {"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{contracts.ModeStrict, 0, contracts.CacheAuto}}, - {"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheBypass}}, + {"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheAuto}}, + {"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5, Cache: contracts.CacheRefresh}}, + {"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheAuto}}, + {"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -75,13 +75,13 @@ func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) { if err != nil { t.Fatal(err) } - if resolved.ChunkControls != (contracts.ProducerControls{contracts.ModePermissive, 0, contracts.CacheBypass}) { + if resolved.ChunkControls != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) { t.Fatalf("chunk controls = %#v", resolved.ChunkControls) } for _, step := range resolved.Steps { for _, lane := range step.ArtifactLanes { for _, got := range []contracts.ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} { - if got != (contracts.ProducerControls{contracts.ModePermissive, 2, contracts.CacheBypass}) { + if got != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2, Cache: contracts.CacheBypass}) { t.Fatalf("lane controls = %#v", got) } }