From 772b8619c1bf02a45629ad24db9462a78c4cf0b9 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 00:17:58 +0000 Subject: [PATCH] Migrate item normalization to native correction and explicit fallback handling --- .../normalize/prompts/prompt.yaml | 18 +- docs/roadmap/implementation.md | 23 ++ internal/framework/pipeline/producer.go | 2 +- .../framework/pipeline/producer_outcome.go | 31 +- .../pipeline/producer_outcome_test.go | 56 +++- .../framework/pipeline/runner_operation.go | 12 +- .../itemregistry/native_execution_test.go | 248 ++++++++++++++ .../dnd/normalize/itemregistry/normalizer.go | 260 ++++++++------- .../normalize/itemregistry/normalizer_test.go | 311 +++++++++++------- .../normalize/itemregistry/prompt_assets.go | 78 ----- .../itemregistry/shape/validator_test.go | 2 +- 11 files changed, 704 insertions(+), 337 deletions(-) create mode 100644 internal/modules/dnd/normalize/itemregistry/native_execution_test.go delete mode 100644 internal/modules/dnd/normalize/itemregistry/prompt_assets.go diff --git a/assets/dnd/item-registry/normalize/prompts/prompt.yaml b/assets/dnd/item-registry/normalize/prompts/prompt.yaml index 4a2b0b02..798c4154 100644 --- a/assets/dnd/item-registry/normalize/prompts/prompt.yaml +++ b/assets/dnd/item-registry/normalize/prompts/prompt.yaml @@ -11,9 +11,9 @@ prompts: messages: - role: system content_file: ../../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../../../generic/normalize/deduplication/prompts/protocol.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -27,3 +27,17 @@ prompts: format: json validation_mode: json_schema schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json + + - id: dnd.item_registry.normalize.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ../../../../generic/normalize/deduplication/prompts/correction.md + output: + format: json + validation_mode: json_schema + schema_path: generic/normalize/deduplication/schemas/semantic_reconciliation_llm.v1.json diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 22b23465..6f165a28 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -625,6 +625,29 @@ as a request to restart the core. ## Stage 14: Migrate Item Registry Normalization +**Status:** Complete. Item normalization now uses one fixed preprocessing and +reconciliation preparation, native initial/correction requests, and the shared +producer helper. Each response applies its own safe proposal to original records +before operation-scoped assessment. Currency compatibility, identity derivation, +evidence ordering, and domain diagnostics remain. Added the same-schema/default- +profile correction declaration and static dependencies; removed module retries, +completion/asset registration, and manifest/fingerprint metadata. +Permissive output uses only the final safe candidate or the supplied merged +artifact after a result-less failure. Fallback construction checks codec, registry +identity, and source usability without ordinary revalidation. Preparation limits +honor resolved mode; insufficient candidates use the existing assessment path +without native requests. Exposed/factored the existing terminal/no-model assessment +functions for these real callers rather than duplicating acceptance policy. +Retained domain regressions and added native-fake correction, final safe partial, +unchanged-candidate, failed-continuation/no-earlier-recovery, structural-failure, +mode/limit, no-model, fallback-usability, override, and cancellation tests. +Focused pipeline tests/vet/race, terminal tests, complete reconciliation/asset and +item-shape tests/vet/race, production framework builds, and `git diff --check` +pass. Item-normalizer/codec/identity/source-validator tests remain blocked at +setup by shared legacy asset helpers importing old ConfigKit's removed YAML +`plugin/limit`; the new module tests are not claimed passing. Their full execution +remains required after scheduled dependency cleanup and in final verification. + **Goal:** prove a complete native normalization operation and safe fallback. **Inspect/change:** `internal/modules/dnd/normalize/itemregistry`, its validators/ diff --git a/internal/framework/pipeline/producer.go b/internal/framework/pipeline/producer.go index 106c96cf..ece91281 100644 --- a/internal/framework/pipeline/producer.go +++ b/internal/framework/pipeline/producer.go @@ -155,7 +155,7 @@ func RunProducer[T any]( if fallback != nil { construct = func(ctx context.Context) (T, error) { return fallback(ctx, candidate) } } - return finishProducerFailure(ctx, operation.Controls.Mode, failure, construct) + return FinishProducerFailure(ctx, operation.Controls.Mode, failure, construct) } // producerRequest applies native options to a value copy. The invocation diff --git a/internal/framework/pipeline/producer_outcome.go b/internal/framework/pipeline/producer_outcome.go index 07ecd73b..94a6e660 100644 --- a/internal/framework/pipeline/producer_outcome.go +++ b/internal/framework/pipeline/producer_outcome.go @@ -8,9 +8,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -// finishProducerFailure applies the resolved terminal mode without retrying +// FinishProducerFailure applies the resolved terminal mode without retrying // production or running ordinary validators against a supported fallback. -func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) { +func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionMode, cause error, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) { var zero T if ctx == nil { return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil") @@ -40,3 +40,30 @@ func finishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer mode %q", mode) } } + +// AssessProducer assesses a no-model result once, including deterministic +// short-circuits inside otherwise model-backed modules. It never constructs a +// native request or gives deterministic work a correction loop. +func AssessProducer[T any](ctx context.Context, operation contracts.ProducerOperation[T], value T, fallback func(context.Context) (T, error)) (T, contracts.ProducerOutcome, error) { + var zero T + if ctx == nil { + return zero, contracts.ProducerOutcome{}, errors.New("producer context must not be nil") + } + if err := ctx.Err(); err != nil { + return zero, contracts.ProducerOutcome{}, err + } + if operation.Assess == nil { + return zero, contracts.ProducerOutcome{}, errors.New("producer assessment must be supplied") + } + assessment, err := operation.Assess(ctx, value) + if err != nil { + return zero, contracts.ProducerOutcome{}, err + } + if err := ctx.Err(); err != nil { + return zero, contracts.ProducerOutcome{}, err + } + if !assessment.Accepted() { + return FinishProducerFailure(ctx, operation.Controls.Mode, assessment.Err(), fallback) + } + return value, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil +} diff --git a/internal/framework/pipeline/producer_outcome_test.go b/internal/framework/pipeline/producer_outcome_test.go index 419febb7..abdb0663 100644 --- a/internal/framework/pipeline/producer_outcome_test.go +++ b/internal/framework/pipeline/producer_outcome_test.go @@ -12,7 +12,7 @@ func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) { failure := errors.New("candidate not accepted") for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { calls := 0 - value, outcome, err := finishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil }) + value, outcome, err := FinishProducerFailure(context.Background(), mode, failure, func(context.Context) (string, error) { calls++; return "supported", nil }) if mode == contracts.ModeStrict { if !errors.Is(err, failure) || calls != 0 || value != "" || outcome.Disposition != "" { t.Fatalf("strict = %q %#v %v, calls %d", value, outcome, err, calls) @@ -25,11 +25,11 @@ func TestProducerTerminalModeControlsSupportedFallback(t *testing.T) { func TestProducerFallbackCannotHideConstructionFailure(t *testing.T) { original, construction := errors.New("original failure"), errors.New("no invariant-preserving fallback") - _, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction }) + _, outcome, err := FinishProducerFailure(context.Background(), contracts.ModePermissive, original, func(context.Context) (string, error) { return "", construction }) if !errors.Is(err, original) || !errors.Is(err, construction) || outcome.Disposition != "" { t.Fatalf("outcome = %#v, error = %v", outcome, err) } - if _, _, err := finishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) { + if _, _, err := FinishProducerFailure[string](context.Background(), contracts.ModePermissive, original, nil); !errors.Is(err, original) { t.Fatalf("unsupported fallback = %v", err) } } @@ -41,7 +41,7 @@ func TestProducerCancellationNeverDeliversFallback(t *testing.T) { if !cancelDuringFallback { cancel() } - value, outcome, err := finishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) { + value, outcome, err := FinishProducerFailure(ctx, contracts.ModePermissive, errors.New("failure"), func(context.Context) (string, error) { calls++ cancel() return "not delivered", nil @@ -57,8 +57,54 @@ func TestProducerCancellationNeverDeliversFallback(t *testing.T) { } func TestProducerLocalTimeoutDoesNotBecomeCallerCancellation(t *testing.T) { - value, outcome, err := finishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil }) + value, outcome, err := FinishProducerFailure(context.Background(), contracts.ModePermissive, context.DeadlineExceeded, func(context.Context) (string, error) { return "supported", nil }) if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback { t.Fatalf("local timeout = %q %#v %v", value, outcome, err) } } + +func TestNoModelProducerIsAssessedOnceWithoutRevalidatingFallback(t *testing.T) { + for _, approve := range []bool{true, false} { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + assessed, fallbacks := 0, 0 + operation := contracts.ProducerOperation[string]{ + Controls: contracts.ProducerControls{Mode: mode}, ReuseEligible: true, + Assess: func(context.Context, string) (contracts.CandidateAssessment, error) { + assessed++ + if approve { + return contracts.CandidateAssessment{}, nil + } + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "policy", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{Message: "not accepted"}}}}, nil + }, + } + value, outcome, err := AssessProducer(context.Background(), operation, "candidate", func(context.Context) (string, error) { fallbacks++; return "supported", nil }) + if assessed != 1 { + t.Fatalf("assessed %d times", assessed) + } + if approve { + if err != nil || value != "candidate" || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 { + t.Fatalf("acceptance = %q %#v %v", value, outcome, err) + } + } else if mode == contracts.ModeStrict { + if err == nil || value != "" || outcome.Disposition != "" || fallbacks != 0 { + t.Fatalf("strict failure = %q %#v %v", value, outcome, err) + } + } else if err != nil || value != "supported" || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 { + t.Fatalf("fallback = %q %#v %v", value, outcome, err) + } + } + } +} + +func TestNoModelProducerStopsAfterAssessmentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, + Assess: func(context.Context, int) (contracts.CandidateAssessment, error) { + cancel() + return contracts.CandidateAssessment{}, nil + }} + _, outcome, err := AssessProducer(ctx, operation, 1, func(context.Context) (int, error) { t.Fatal("fallback after cancellation"); return 0, nil }) + if !errors.Is(err, context.Canceled) || outcome.Disposition != "" { + t.Fatalf("cancellation = %#v %v", outcome, err) + } +} diff --git a/internal/framework/pipeline/runner_operation.go b/internal/framework/pipeline/runner_operation.go index 1e6c8a22..cc04e1bd 100644 --- a/internal/framework/pipeline/runner_operation.go +++ b/internal/framework/pipeline/runner_operation.go @@ -17,17 +17,7 @@ func completeProducerOperation[T any](ctx context.Context, class contracts.Execu return zero, contracts.ProducerOutcome{}, err } if class == contracts.ExecutionClassDeterministic { - assessment, err := operation.Assess(ctx, value) - if err != nil { - return zero, contracts.ProducerOutcome{}, err - } - if err := ctx.Err(); err != nil { - return zero, contracts.ProducerOutcome{}, err - } - if !assessment.Accepted() { - return finishProducerFailure(ctx, operation.Controls.Mode, assessment.Err(), fallback) - } - outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible} + return AssessProducer(ctx, operation, value, fallback) } else if class != contracts.ExecutionClassLLMBacked { return zero, contracts.ProducerOutcome{}, fmt.Errorf("unsupported producer execution class %q", class) } diff --git a/internal/modules/dnd/normalize/itemregistry/native_execution_test.go b/internal/modules/dnd/normalize/itemregistry/native_execution_test.go new file mode 100644 index 00000000..162d0171 --- /dev/null +++ b/internal/modules/dnd/normalize/itemregistry/native_execution_test.go @@ -0,0 +1,248 @@ +package itemregistry + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" + "go.maximumdirect.net/promptkit" +) + +const safeFirstProposal = `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}` +const invalidItemProposal = `{"duplicate_groups":[{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + +func mergedItems() dnd.ItemRegistry { + doc := semanticDocument() + result := dnd.ItemRegistry{Items: []dnd.Item{ + {Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {Name: "Astral Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + for i := range result.Items { + result.Items[i].ID = identity.DeriveID(result.Items[i].Name) + } + return result +} + +func itemNames(value dnd.ItemRegistry) []string { + names := make([]string, len(value.Items)) + for i, item := range value.Items { + names[i] = item.Name + } + return names +} + +func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) { + finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` + for _, test := range []struct { + name string + mode contracts.ExecutionMode + responses []string + rejectFirst bool + wantError bool + wantFallback bool + wantNames []string + }{ + {"corrected proposal accepted", contracts.ModeStrict, []string{invalidItemProposal, safeFirstProposal}, false, false, false, []string{"Compass of the Stars", "Astral Compass"}}, + {"unchanged candidate reassessed", contracts.ModeStrict, []string{invalidItemProposal, invalidItemProposal}, false, true, false, nil}, + {"final partial proposal only", contracts.ModePermissive, []string{safeFirstProposal, finalPartial}, true, false, true, []string{"Star Compass", "Astral Compass"}}, + } { + t.Run(test.name, func(t *testing.T) { + input := mergedItems() + before := cloneRegistry(input) + client := &recordingNormalizerClient{responses: test.responses} + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = len(test.responses) - 1 + request.SessionID = "normalization-session" + model := "run-wide-normalization-model" + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + base := request.Operation.Assess + var assessed []dnd.ItemRegistry + request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessed = append(assessed, cloneRegistry(value)) + assessment, err := base(ctx, value) + if test.rejectFirst { + finding := contracts.ValidatorAssessment{ValidatorName: "configured-domain-policy", Decision: contracts.AssessmentApproved, Result: contracts.ValidationResult{Approved: true}} + if len(assessed) == 1 { + finding.Decision = contracts.AssessmentRejected + finding.Result = contracts.ValidationResult{Message: "operator-only-rejection", CorrectionGuidance: "Reconsider the first compass grouping."} + } + assessment.Validators = append(assessment.Validators, finding) + } + return assessment, err + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if (err != nil) != test.wantError { + t.Fatalf("result = %#v %v", result, err) + } + if len(client.requests) != len(test.responses) || len(assessed) != len(test.responses) || len(prepared) != len(test.responses) { + t.Fatalf("provider/assessment/preparation = %d/%d/%d, want %d", len(client.requests), len(assessed), len(prepared), len(test.responses)) + } + if !reflect.DeepEqual(input, before) { + t.Fatal("proposal mutated fixed originals") + } + if test.name == "unchanged candidate reassessed" && !reflect.DeepEqual(assessed[0], assessed[1]) { + t.Fatal("fixture did not exercise an unchanged typed candidate") + } + for i, call := range client.requests { + if call.Target.Model != model || call.Prompt.SessionID != "normalization-session" || prepared[i].SelectedProfileID != "item-normalize-test" { + t.Fatal("native override lost") + } + wantID := PromptID + if i > 0 { + wantID = CorrectionPromptID + } + if prepared[i].PromptID != wantID { + t.Fatalf("prompt = %q, want %q", prepared[i].PromptID, wantID) + } + if i == 0 { + continue + } + old := client.requests[i-1].Prompt.Messages + current := call.Prompt.Messages + if len(current) != len(old)+2 || !reflect.DeepEqual(current[:len(old)], old) { + t.Fatal("native continuation changed fixed evidence/history") + } + if current[len(old)].Role != "assistant" || current[len(old)].Content != test.responses[i-1] { + t.Fatal("native continuation lost the prior response") + } + if strings.Contains(current[len(old)+1].Content, "operator-only-rejection") { + t.Fatal("operator diagnostic became model guidance") + } + } + if test.wantError { + return + } + if !reflect.DeepEqual(itemNames(result.Value), test.wantNames) || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || result.Outcome.ReuseEligible == test.wantFallback { + t.Fatalf("final outcome = %#v", result) + } + if _, err := itemcodec.New().Encode(result.Value); err != nil { + t.Fatalf("durable artifact encoding: %v", err) + } + if test.wantFallback { + refs := result.Value.Items[1].SourceRefs + if len(refs) != 2 || refs[0].StartUnitID != 20 || refs[1].StartUnitID != 30 { + t.Fatalf("final proposal carried earlier evidence: %#v", refs) + } + } + }) + } +} + +func TestNativeFailureUsesMergedInputWithoutRecoveringEarlierProposal(t *testing.T) { + failure := errors.New("provider unavailable") + for _, failCorrection := range []bool{false, true} { + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + input := mergedItems() + client := &recordingNormalizerClient{err: failure} + if failCorrection { + client.err = nil + client.responses = []string{safeFirstProposal} + client.errors = []error{nil, failure} + } + request := normalizeRequestWithSource(input, semanticDocument()) + request.Operation.Controls.Mode = mode + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "policy", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{CorrectionGuidance: "Keep the distinct compass records."}}}}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + wantCalls, wantAssess := 1, 0 + if failCorrection { + wantCalls, wantAssess = 2, 1 + } + if len(client.requests) != wantCalls || assessments != wantAssess { + t.Fatalf("calls = %d/%d, want %d/%d", len(client.requests), assessments, wantCalls, wantAssess) + } + if mode == contracts.ModeStrict { + if !errors.Is(err, failure) { + t.Fatalf("strict failure = %v", err) + } + } else if err != nil || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) { + t.Fatalf("fallback recovered an earlier proposal: %#v %v", result, err) + } + } + } +} + +func TestNativeStructuralFailureProducesNoSemanticCandidate(t *testing.T) { + input := mergedItems() + client := &recordingNormalizerClient{response: `{"duplicate_groups":null}`} + request := permissiveRequestWithSource(input, semanticDocument()) + request.Operation.Controls.MaxAdditionalTurns = 2 + assessed := 0 + request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if err != nil || assessed != 0 || !reflect.DeepEqual(result.Value, input) || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible { + t.Fatalf("native structural failure = %#v %v, assessed %d", result, err, assessed) + } + for _, call := range client.requests { + for _, message := range call.Prompt.Messages { + if strings.Contains(message.Content, "Revise the previous duplicate-entity proposal using these findings:") { + t.Fatal("native structural failure started semantic continuation") + } + } + } +} + +func TestInsufficientCandidatesUseAssessmentWithoutNativeExecution(t *testing.T) { + input := mergedItems() + input.Items = input.Items[:1] + client := &recordingNormalizerClient{} + request := normalizeRequestWithSource(input, semanticDocument()) + base := request.Operation.Assess + assessed := 0 + request.Operation.Assess = func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessed++ + return base(ctx, value) + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if err != nil || len(client.requests) != 0 || assessed != 1 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) { + t.Fatalf("no-model outcome = %#v %v, assessed %d", result, err, assessed) + } +} + +func TestCancellationNeverDeliversNormalizationFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &recordingNormalizerClient{onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newNormalizer(t, client).Normalize(ctx, permissiveRequestWithSource(mergedItems(), semanticDocument())) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("cancellation = %#v %v, calls %d", result, err, len(client.requests)) + } +} + +func TestFallbackCannotPublishAnUnusableMergedRegistry(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mutate func(*dnd.ItemRegistry) + }{ + {"identity mismatch", func(value *dnd.ItemRegistry) { value.Items[0].ID = identity.DeriveID("Wrong identity") }}, + {"foreign source", func(value *dnd.ItemRegistry) { value.Items[0].SourceRefs[0].SourceID = "other-session" }}, + } { + t.Run(test.name, func(t *testing.T) { + input := mergedItems() + test.mutate(&input) + result, err := newNormalizer(t, &recordingNormalizerClient{err: failure}).Normalize(context.Background(), permissiveRequestWithSource(input, semanticDocument())) + if !errors.Is(err, failure) || result.Outcome.Disposition != "" { + t.Fatalf("invalid fallback = %#v %v", result, err) + } + }) + } +} diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer.go b/internal/modules/dnd/normalize/itemregistry/normalizer.go index e2dec80b..669346c7 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer.go @@ -14,24 +14,23 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + "go.maximumdirect.net/promptkit" ) const ( - Key = "dnd/item-registry" - PromptID = "dnd.item_registry.normalize" - PromptVersion = "v1" - normalizationPolicy = "dnd.item_registry.normalize.v4" - NormalizationPolicy = normalizationPolicy + Key = "dnd/item-registry" + PromptID = "notarius/dnd.item_registry.normalize" + CorrectionPromptID = "notarius/dnd.item_registry.normalize.repair" ReasonCodeItemFieldsNormalized = "item_fields_normalized" ReasonCodeItemIDRecomputed = "item_id_recomputed" ReasonCodeSourceReferencesNormalized = "source_references_normalized" ReasonCodeDuplicateItemCollapsed = "duplicate_item_collapsed" ReasonCodeItemSemanticProposalInvalid = "item_semantic_proposal_invalid" - ReasonCodeItemSemanticRetryProposalInvalid = "item_semantic_retry_proposal_invalid" ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted" ) @@ -39,28 +38,16 @@ var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.ItemRegistry] = (*Normalizer)(nil) -var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} type Normalizer struct { - engine *semanticreconcile.Engine + engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) { - if llmClient == nil { - return nil, normalizerErrorf("LLM client must not be nil") - } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, normalizerErrorf("load prompt metadata: %w", err) - } - engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{ - ID: PromptID, Version: PromptVersion, SHA256: promptSHA, - }, semanticreconcile.DefaultLimits()) - if err != nil { - return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err) +func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) { + if engine == nil { + return nil, normalizerErrorf("native engine must not be nil") } return &Normalizer{engine: engine}, nil } @@ -68,33 +55,12 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (n *Normalizer) ManifestMetadata() map[string]any { - if n == nil || n.engine == nil { - return nil - } - metadata := n.engine.ManifestMetadata() - metadata["identity_policy"] = identity.Policy - metadata["normalization_policy"] = normalizationPolicy - return metadata -} - -func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if n == nil || n.engine == nil { - return nil - } - fingerprints := n.engine.CheckpointFingerprints() - return append(fingerprints, - pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy}, - pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy}, - ) -} - func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemRegistry]) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { if n == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalizer must not be nil") } if n.engine == nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil") + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("native engine must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context must not be nil") @@ -106,78 +72,145 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil") } + // Keep the already available merged artifact separate from preprocessed + // originals. A failed native response must not recover an earlier proposal. + merged := cloneRegistry(req.MergeOutput.Value) order := shared.NewSourceRefOrder(req.Source) - records, findings := preprocessRecords(req.MergeOutput.Value, order) + records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) - if len(records) < 2 { - return normalizationResult(deterministic, findings, nil, nil) - } + findings := append([]diagnostics.Finding(nil), baseFindings...) + var advisories []diagnostics.Finding + discardedGroups := -1 + var omissionDetails []string + fallback := func(_ context.Context, final *dnd.ItemRegistry) (dnd.ItemRegistry, error) { + value := cloneRegistry(merged) + if final != nil { + // Each interpreted candidate was built afresh from fixed original records, + // using only the final proposal's compatible, non-overlapping safe groups. + value = cloneRegistry(*final) + } else { + findings, advisories = nil, nil + discardedGroups = -1 + omissionDetails = nil + } + if err := validateFallback(value, req.Source); err != nil { + return dnd.ItemRegistry{}, err + } + return value, nil + } + finish := func(value dnd.ItemRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { + if err != nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalize items: %w", err) + } + result, err := normalizationResult(value, findings, advisories) + if err != nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err + } + result.Outcome = outcome + if outcome.Disposition == contracts.ProducerFallback { + details := append([]string(nil), omissionDetails...) + details = append(details, outcome.FallbackReason) + groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) + if err != nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) + } + result.Diagnostics = append(result.Diagnostics, groups...) + } + return result, nil + } + noModel := func() (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { + value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) + } + if len(records) < 2 { + return noModel() + } candidates, envelopes, err := reconciliationInputs(records) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err) } - reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{ - StageName: Key, Source: req.Source, Candidates: candidates, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction, - }) + preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits()) if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err) + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } - - switch reconciliation.Disposition() { - case semanticreconcile.SkippedInsufficientCandidates: - return normalizationResult(deterministic, findings, nil, nil) - case semanticreconcile.SkippedLimitExceeded: - return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits")) - case semanticreconcile.RetryableInvalidStructuredOutput: - return n.invalidStructuredResult(deterministic, findings) - case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups: - default: - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition()) + switch preparation.Disposition() { + case semanticreconcile.InsufficientCandidates: + return noModel() + case semanticreconcile.LimitExceeded: + value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) }) + return finish(value, outcome, err) } - - applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order) + initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID) if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err) + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build native request: %w", err) } - findings = append(findings, semanticFindings...) - discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups - if discardedGroups == 0 { - return normalizationResult(recordList(applied), findings, advisoryFindings, reconciliation.ModelCandidate()) - } - return retryResult(recordList(applied), findings, advisoryFindings, reconciliation, rejectedGroups) + value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemRegistry, []string, error) { + response, err := semanticreconcile.DecodeProposal(result.RawOutput) + if err != nil { + return nil, nil, err + } + assessment := preparation.Assess(response) + applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) + if err != nil { + return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) + } + findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...) + advisories = advisoryFindings + discardedGroups = assessment.DiscardedGroupCount() + rejectedGroups + omissionDetails = semanticreconcile.IssueDetails(assessment.Issues()) + guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange()) + if err != nil { + return nil, nil, err + } + if rejectedGroups > 0 { + guidance = append(guidance, "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items.") + omissionDetails = append(omissionDetails, "currency may only be consolidated with aliases of one denomination") + } + candidate := recordList(applied) + return &candidate, guidance, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil + }, fallback) + return finish(value, outcome, err) } -func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { - return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract")) +// validateFallback checks the durable artifact and the identity/source invariants +// used by registry consumers, not the configured ordinary validator chain. +func validateFallback(value dnd.ItemRegistry, document *source.SourceDocument) error { + if _, err := itemcodec.New().Encode(value); err != nil { + return normalizerErrorf("construct item fallback: %w", err) + } + if issues := identity.ValidateList(value); len(issues) > 0 { + return normalizerErrorf("construct item fallback: registry identity is not usable downstream") + } + index := source.NewDocumentIndex(document) + for _, item := range value.Items { + for _, reference := range item.SourceRefs { + if err := index.ValidateRef(reference); err != nil { + return normalizerErrorf("construct item fallback: %w", err) + } + } + } + return nil } -func retryResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, reconciliation semanticreconcile.Result, rejectedGroups int) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { - issues := reconciliation.Issues() - operatorDetails := semanticreconcile.IssueDetails(issues) - correctionDetails, err := reconciliation.CorrectionDetails() - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction details: %w", err) +func cloneRegistry(value dnd.ItemRegistry) dnd.ItemRegistry { + if value.Items == nil { + return dnd.ItemRegistry{} } - var additionalGuidance []string - if rejectedGroups > 0 { - const currencyGuidance = "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items." - operatorDetails = append(operatorDetails, "currency may only be consolidated with aliases of one denomination") - correctionDetails = append(correctionDetails, currencyGuidance) - additionalGuidance = append(additionalGuidance, currencyGuidance) + result := dnd.ItemRegistry{Items: make([]dnd.Item, len(value.Items))} + for i, item := range value.Items { + result.Items[i] = cloneItem(item) } - correctionGuidance, err := reconciliation.CorrectionGuidance(additionalGuidance...) - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err) - } - discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups - return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", operatorDetails), semanticFallbackFinding(discardedGroups, correctionDetails...)) + return result } func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding { - message := "semantic reconciliation could not be completed; all affected candidates were preserved separately" - if discarded >= 0 { + message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output" + if discarded > 0 { message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded) } if len(details) > 0 { @@ -186,8 +219,8 @@ func semanticFallbackFinding(discarded int, details ...string) diagnostics.Findi return diagnostics.Finding{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message} } -func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { - diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) +func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { + groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err) } @@ -195,34 +228,7 @@ func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []di if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect data-quality diagnostics: %w", err) } - diagnosticGroups = append(diagnosticGroups, advisoryGroups...) - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil -} - -func fallbackResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { - result, err := normalizationResult(value, findings, advisoryFindings, nil) - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Diagnostics = append(result.Diagnostics, fallbackGroups...) - return result, nil -} - -func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate, reasonCode, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { - result, err := normalizationResult(value, findings, advisoryFindings, candidate) - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err - } - fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) - if err != nil { - return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) - } - result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups} - return result, nil + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: append(groups, advisoryGroups...)}, nil } type normalizedRecord struct { @@ -363,7 +369,7 @@ func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding { func itemScope(index int) string { return fmt.Sprintf("items[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { - return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind} + return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind} } func Register(registry *pipeline.NormalizerRegistry) error { return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemRegistry], error) { @@ -371,7 +377,7 @@ func Register(registry *pipeline.NormalizerRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options) + return New(request.Dependencies.Engine, options) }) } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer_test.go b/internal/modules/dnd/normalize/itemregistry/normalizer_test.go index 6f2eb436..68b73d19 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer_test.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer_test.go @@ -3,28 +3,26 @@ package itemregistry import ( "context" "encoding/json" - "errors" "fmt" "reflect" "strconv" "strings" "testing" - "time" + "gitea.maximumdirect.net/eric/notarius/assets" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" ) -func TestModuleContractAndMetadata(t *testing.T) { - want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind} +func TestModuleContractAndNativeDependencies(t *testing.T) { + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } @@ -33,17 +31,12 @@ func TestModuleContractAndMetadata(t *testing.T) { t.Fatal(err) } if _, err := New(nil, Options{}); err == nil { - t.Fatal("New() accepted nil client") + t.Fatal("New() accepted nil native engine") } - metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata() - limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any) - if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 { - t.Fatalf("metadata = %#v", metadata) - } - for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} { - if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) { - t.Fatalf("fingerprints missing %q", name) - } + got := ModuleSpec() + got.Prompts[0] = "caller mutation" + if !reflect.DeepEqual(ModuleSpec(), want) { + t.Fatal("static dependency declaration is not owned") } } @@ -93,7 +86,7 @@ func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing Label string `json:"label"` } `json:"candidates"` } - if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" { + if err := json.Unmarshal(renderedInput(t, client.requests[0], "candidates"), &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" { t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err) } repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc)) @@ -125,14 +118,14 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) { }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry != nil || len(result.Value.Items) != 2 { + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Items) != 2 { t.Fatalf("Normalize() = %#v, %v", result, err) } merged := result.Value.Items[0] if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) { t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics) } - encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content) + encoded := string(renderedInput(t, client.requests[0], "candidates")) + string(renderedInput(t, client.requests[0], "transcript")) if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) { t.Fatalf("private inputs = %s", encoded) } @@ -149,7 +142,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { response string wantNames []string wantRefCounts []int - wantRetry bool + wantFallback bool reasonCode string disposition contracts.DiagnosticDisposition }{ @@ -175,7 +168,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`, wantNames: []string{"Gold Pieces", "Silver Pieces"}, wantRefCounts: []int{1, 1}, - wantRetry: true, + wantFallback: true, reasonCode: ReasonCodeItemSemanticProposalInvalid, disposition: contracts.DiagnosticDispositionAdvisory, }, @@ -188,7 +181,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`, wantNames: []string{"Gold Pieces", "Longsword"}, wantRefCounts: []int{1, 1}, - wantRetry: true, + wantFallback: true, reasonCode: ReasonCodeItemSemanticProposalInvalid, disposition: contracts.DiagnosticDispositionAdvisory, }, @@ -213,7 +206,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`, wantNames: []string{"Gold Pieces", "Longsword"}, wantRefCounts: []int{1, 1}, - wantRetry: true, + wantFallback: true, reasonCode: ReasonCodeItemSemanticProposalInvalid, disposition: contracts.DiagnosticDispositionAdvisory, }, @@ -228,8 +221,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { before.Items[index] = cloneItem(item) } - result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) { + result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), permissiveRequestWithSource(input, doc)) + if err != nil || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) { t.Fatalf("Normalize() = %#v, %v", result, err) } if len(result.Value.Items) != len(test.wantNames) { @@ -240,8 +233,8 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index]) } } - if test.wantRetry && (len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted || result.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid) { - t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry) + if test.wantFallback && (result.Outcome.ReuseEligible || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning)) { + t.Fatalf("fallback lacks qualification: %#v", result) } }) } @@ -255,9 +248,9 @@ func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) { {Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,3],"canonical_candidate_number":99}]}`} - result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 { - t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err) + result, err := newNormalizer(t, client).Normalize(context.Background(), permissiveRequestWithSource(input, doc)) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || len(result.Value.Items) != 3 || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) { + t.Fatalf("Normalize() = %#v, %v; want qualified safe fallback", result, err) } } @@ -274,9 +267,11 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) { {Name: "Rope", SourceRefs: ref(10)}, }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,4],"canonical_candidate_number":3},{"candidate_numbers":[5,99],"canonical_candidate_number":5}]}`} - result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry == nil { - t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err) + request := permissiveRequestWithSource(input, doc) + request.Operation.Controls.MaxAdditionalTurns = 1 + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback { + t.Fatalf("Normalize() = %#v, %v; want qualified safe partial output", result, err) } wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"} if len(result.Value.Items) != len(wantNames) { @@ -290,115 +285,214 @@ func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) { if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) { t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics) } - if !strings.Contains(result.Retry.CorrectionGuidance, "aliases of the same denomination") || !strings.Contains(result.Retry.CorrectionGuidance, "do not combine currency with non-currency items") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 5") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeItemSemanticRetryProposalInvalid) { - t.Fatalf("correction guidance = %q, want shared and item-specific prose only", result.Retry.CorrectionGuidance) + + if len(client.requests) != 2 { + t.Fatalf("native calls = %d, want initial and correction", len(client.requests)) } - if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "same denomination") { - t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry) + messages := client.requests[1].Prompt.Messages + guidance := messages[len(messages)-1].Content + for _, expected := range []string{"aliases of the same denomination", "do not combine currency with non-currency items", "candidate-number range 1 through 5", "source positions"} { + if !strings.Contains(guidance, expected) { + t.Fatalf("missing domain/shared guidance %q", expected) + } + } + if strings.Contains(guidance, "member_unknown") || strings.Contains(guidance, ReasonCodeItemSemanticProposalInvalid) { + t.Fatal("internal diagnostics copied into correction") + } + var warning string + for _, group := range result.Diagnostics { + if group.ReasonCode == ReasonCodeItemSemanticReconciliationExhausted && len(group.Samples) > 0 { + warning = group.Samples[0].Message + } + } + if !strings.Contains(warning, "2 proposal group(s)") || !strings.Contains(warning, "preserved separately") || !strings.Contains(warning, "same denomination") { + t.Fatalf("fallback warning = %q", warning) } } -func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) { - client := &recordingNormalizerClient{} +func TestPreparationLimitsFollowResolvedModeWithoutModelOrAssessment(t *testing.T) { doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}} limit := semanticreconcile.DefaultLimits().MaximumCandidates input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)} - for index := range input.Items { - input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + for i := range input.Items { + name := fmt.Sprintf("Item %d", i) + input.Items[i] = dnd.Item{ID: identity.DeriveID(name), Name: name, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} } - result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || result.Retry != nil { - t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err) - } - if len(client.requests) != 0 || len(result.Value.Items) != limit+1 { - t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items)) - } - if !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) { - t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics) + for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { + t.Run(string(mode), func(t *testing.T) { + client := &recordingNormalizerClient{} + request := normalizeRequestWithSource(input, doc) + request.Operation.Controls.Mode = mode + assessed := 0 + request.Operation.Assess = func(context.Context, dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + assessed++ + return contracts.CandidateAssessment{}, nil + } + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if len(client.requests) != 0 || assessed != 0 { + t.Fatal("unready preparation caused model execution or ordinary validation") + } + if mode == contracts.ModeStrict { + if err == nil { + t.Fatal("strict preparation-limit failure succeeded") + } + } else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || !reflect.DeepEqual(result.Value, input) { + t.Fatalf("permissive preparation limit: %#v %v", result, err) + } + }) } } -func TestNormalizeRetryFallbackErrorsAndIdempotence(t *testing.T) { +func TestNormalizationRemainsIdempotent(t *testing.T) { doc := semanticDocument() input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}} - invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid { - t.Fatalf("invalid result = %#v, %v", invalid, err) - } - if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") { - t.Fatalf("invalid structured retry = %#v, want feedback-free retry with final-cause fallback", invalid.Retry) - } - _, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - if err == nil || !strings.Contains(err.Error(), "provider unavailable") { - t.Fatalf("provider error = %v", err) - } first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) - second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc)) - if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) { - t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr) - } -} - -func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := semanticreconcile.RegisterAssets(registry); err != nil { - t.Fatal(err) - } - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "item-normalize-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}}) if err != nil { t.Fatal(err) } - if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral { - t.Fatalf("prepared prompt = %#v", prepared) + second, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc)) + if err != nil || !reflect.DeepEqual(first.Value, second.Value) { + t.Fatalf("repeated normalization changed output: %#v %v", second, err) } - if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) { - t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared) +} + +func TestNativeNormalizationPromptPairPreparesWithSameContract(t *testing.T) { + client := &recordingNormalizerClient{} + normalizer := newNormalizer(t, client) + var initial promptkit.PreparedRun + for _, id := range []string{PromptID, CorrectionPromptID} { + inputs := map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep currency denominations separate.")} + if id == PromptID { + inputs = map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`), "transcript": promptkit.Inline(`{"windows":[]}`)} + } + handle, err := normalizer.engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: id, APIKey: "test-only", Inputs: inputs}) + if err != nil { + t.Fatal(err) + } + prepared := handle.Details() + handle.Discard() + if prepared.SelectedProfileID != "notarius/dnd-extraction" || prepared.StructuredOutput == nil { + t.Fatal("missing native defaults/schema") + } + if id == PromptID { + initial = prepared + var system string + for _, message := range prepared.Messages { + if message.Role == "system" { + system += message.Content + } + } + for _, rule := range []string{"candidate_number_range", "start_unit_id", "currency denominations", "materially different item", "canonical_candidate_number"} { + if !strings.Contains(system, rule) { + t.Fatalf("missing enduring rule %q", rule) + } + } + } else if !reflect.DeepEqual(initial.OutputContract, prepared.OutputContract) || len(prepared.Messages) != 1 || prepared.Messages[0].Role != "user" || !strings.Contains(prepared.Messages[0].Content, "Keep currency denominations separate.") { + t.Fatal("correction did not preserve the output contract or supply only findings/assignment") + } + } + if len(client.requests) != 0 { + t.Fatal("offline preparation invoked generation") } } type recordingNormalizerClient struct { - response string - err error - requests []contracts.StructuredCompletionRequest + response string + responses []string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { - snapshot, err := contracts.CloneStructuredCompletionRequest(request) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err) +func (c *recordingNormalizerClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(c.requests) + c.requests = append(c.requests, request) + if c.onGenerate != nil { + c.onGenerate(ctx, request) } - c.requests = append(c.requests, snapshot) if c.err != nil { - return contracts.StructuredCompletionResponse{}, c.err + return nil, c.err + } + if index < len(c.errors) && c.errors[index] != nil { + return nil, c.errors[index] } response := c.response + if index < len(c.responses) { + response = c.responses[index] + } if response == "" { response = `{"duplicate_groups":[]}` } - content := []byte(response) - if err := json.Unmarshal(content, output); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return &promptkit.GenerateResponse{Content: response}, nil } -func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer { +func newNormalizer(t *testing.T, client promptkit.LLMClient) *Normalizer { t.Helper() - normalizer, err := New(client, Options{}) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("normalizer fixture", []byte(`promptkit: + profiles: + - id: item-normalize-test + endpoint: https://example.invalid/v1 + model: normalization-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + normalizer, err := New(engine, Options{}) + if err != nil { + t.Fatal(err) } return normalizer } + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := "candidate_number_range" + if name == "transcript" { + key = "windows" + } + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{ - Source: &source.SourceDocument{}, + Source: &source.SourceDocument{}, + LLMProfile: "item-normalize-test", + Operation: contracts.ProducerOperation[dnd.ItemRegistry]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(ctx context.Context, value dnd.ItemRegistry) (contracts.CandidateAssessment, error) { + result, err := identityvalidator.New(identityvalidator.Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value, Stage: string(pipeline.StageNormalize)}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentApproved + if !result.Approved { + decision = contracts.AssessmentRejected + } + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: identityvalidator.Key, Decision: decision, Result: result}}}, nil + }}, MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}, } } @@ -419,11 +513,8 @@ func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, di return false } -func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool { - for _, fingerprint := range fingerprints { - if fingerprint.Name == name && fingerprint.Value != "" { - return true - } - } - return false +func permissiveRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { + request := normalizeRequestWithSource(value, doc) + request.Operation.Controls.Mode = contracts.ModePermissive + return request } diff --git a/internal/modules/dnd/normalize/itemregistry/prompt_assets.go b/internal/modules/dnd/normalize/itemregistry/prompt_assets.go deleted file mode 100644 index 4623172e..00000000 --- a/internal/modules/dnd/normalize/itemregistry/prompt_assets.go +++ /dev/null @@ -1,78 +0,0 @@ -package itemregistry - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -func promptAssetManifest() (shared.PromptAssetManifest, error) { - sharedFiles, err := semanticreconcile.SharedPromptFiles() - if err != nil { - return shared.PromptAssetManifest{}, fmt.Errorf("load shared semantic reconciliation prompt assets: %w", err) - } - return shared.PromptAssetManifest{ - ModuleDir: PromptID, - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{"common-dnd-system.md"}, - ExternalSharedFiles: sharedFiles, - }, nil -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/normalize") - if err != nil { - return nil, fmt.Errorf("scope item normalization assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - manifest, err := promptAssetManifest() - if err != nil { - return err - } - promptFS, err := manifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare item normalization prompt assets: %w", err) - } - return registry.RegisterPromptFS(promptFS, promptAssetRoot) -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - manifest, err := promptAssetManifest() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = manifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/validate/itemregistry/shape/validator_test.go b/internal/modules/dnd/validate/itemregistry/shape/validator_test.go index 24c14180..a1fcf560 100644 --- a/internal/modules/dnd/validate/itemregistry/shape/validator_test.go +++ b/internal/modules/dnd/validate/itemregistry/shape/validator_test.go @@ -35,7 +35,7 @@ func TestValidatorApprovesAndRegisters(t *testing.T) { if err := Register(registry); err != nil { t.Fatalf("Register() error = %v", err) } - if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic { t.Fatalf("registered spec = %#v, ok = %t", got, ok) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {