From bb81354e21d73cd293f7a98d0879973c3313712e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 11 Nov 2026 04:11:34 +0000 Subject: [PATCH] Define correction repair response and conversation replay --- docs/roadmap/implementation.md | 2 + internal/framework/llm/conversation_test.go | 150 ++++++++++++++++++ .../framework/proposal_generation/repair.go | 31 ++++ .../proposal_generation/repair_test.go | 35 ++++ .../assets/correction_repair.v1.json | 57 +++++++ internal/framework/responseschema/registry.go | 5 + .../framework/responseschema/registry_test.go | 4 +- 7 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 internal/framework/llm/conversation_test.go create mode 100644 internal/framework/proposal_generation/repair.go create mode 100644 internal/framework/proposal_generation/repair_test.go create mode 100644 internal/framework/responseschema/assets/correction_repair.v1.json diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index abf43e8..fdaa704 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -15,6 +15,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery) ## Stage 1 — Define the repair response and prove native conversation composition +**Completed.** + 1. In `internal/framework/responseschema`, register `CorrectionRepairKey` with schema ID `audita.correction_repair`, name `audita_correction_repair_v1`, and asset `correction_repair.v1.json`. Use a closed JSON object with three diff --git a/internal/framework/llm/conversation_test.go b/internal/framework/llm/conversation_test.go new file mode 100644 index 0000000..e90e236 --- /dev/null +++ b/internal/framework/llm/conversation_test.go @@ -0,0 +1,150 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" +) + +func TestNativeConversationReplayWithRepairContract(t *testing.T) { + dir := t.TempDir() + files := map[string]string{ + "config.yml": "version: 3\n", + "prompts.yml": `profiles: + - id: proposal-profile + endpoint: https://example.invalid/v1 + model: proposal-model + - id: repair-profile + endpoint: https://example.invalid/v1 + model: repair-model +prompts: + - id: proposal-fixture + messages: + - role: system + content: 'System {{.Subject}}' + - role: user + content: 'Proposal {{.Subject}}' + output: + format: json + validation_mode: json_schema + schema_path: correction_set.v1.json + - id: repair-fixture + messages: + - role: user + content: 'Repair {{.Finding}}' + output: + format: text + validation_mode: none +`, + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") + if err != nil { + t.Fatal(err) + } + var requests []struct { + Model string `json:"model"` + Messages []struct{ Role, Content string } `json:"messages"` + ResponseFormat map[string]any `json:"response_format"` + } + outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`} + transport := transportFunc(func(r *http.Request) (*http.Response, error) { + var request struct { + Model string `json:"model"` + Messages []struct{ Role, Content string } `json:"messages"` + ResponseFormat map[string]any `json:"response_format"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + requests = append(requests, request) + content, _ := json.Marshal(outputs[len(requests)-1]) + body := fmt.Sprintf(`{"model":%q,"choices":[{"message":{"content":%s}}]}`, request.Model, content) + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) + runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + base := pk.RunRequest{PromptID: "proposal-fixture", ProfileID: "proposal-profile", Vars: map[string]string{"Subject": "source"}} + original, err := runtime.Engine.PrepareExecution(ctx, base) + if err != nil { + t.Fatal(err) + } + prefix := original.Details().Messages + initial, err := runtime.Engine.RunPrepared(ctx, original) + if err != nil { + t.Fatal(err) + } + suffix := []pk.RenderedMessage{{Role: pk.RoleAssistant, Content: initial.RawOutput}} + contract, err := OutputContract(responseschema.CorrectionRepairKey, 0) + if err != nil { + t.Fatal(err) + } + for _, finding := range []string{"first", "second"} { + instruction, err := runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: "repair-fixture", ProfileID: "repair-profile", Vars: map[string]string{"Finding": finding}}) + if err != nil { + t.Fatal(err) + } + suffix = append(suffix, instruction.Messages...) + req := base + req.ProfileID = "repair-profile" + req.AppendedMessages = suffix + req.Validation = contract + handle, err := runtime.Engine.PrepareExecution(ctx, req) + if err != nil { + t.Fatal(err) + } + details := handle.Details() + if !reflect.DeepEqual(details.Messages[:len(prefix)], prefix) || !reflect.DeepEqual(details.Messages[len(prefix):], suffix) { + t.Fatalf("prepared conversation = %#v", details.Messages) + } + result, err := runtime.Engine.RunPrepared(ctx, handle) + if err != nil { + t.Fatal(err) + } + if !result.Validation.IsValid { + t.Fatalf("invalid result: %+v", result.Validation) + } + suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: result.RawOutput}) + } + if len(requests) != 3 || requests[0].Model != "proposal-model" || requests[1].Model != "repair-model" || requests[2].Model != "repair-model" { + t.Fatalf("models: %+v", requests) + } + for i, wantLen := range []int{2, 4, 6} { + req := requests[i] + if len(req.Messages) != wantLen { + t.Fatalf("request %d messages: %+v", i, req.Messages) + } + for j, message := range req.Messages { + if message.Role != string(append(prefix, suffix...)[j].Role) || message.Content != append(prefix, suffix...)[j].Content { + t.Fatalf("request %d message %d: %+v", i, j, message) + } + } + encoded, _ := json.Marshal(req.ResponseFormat) + expected := "corrections" + if i > 0 { + expected = "proposal_ref" + } + if !strings.Contains(string(encoded), expected) { + t.Fatalf("request %d contract: %s", i, encoded) + } + } +} diff --git a/internal/framework/proposal_generation/repair.go b/internal/framework/proposal_generation/repair.go new file mode 100644 index 0000000..b2da5f8 --- /dev/null +++ b/internal/framework/proposal_generation/repair.go @@ -0,0 +1,31 @@ +package proposal_generation + +import ( + "encoding/json" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" +) + +// RepairResponse is one model response for an original proposal reference. +type RepairResponse struct { + ProposalRef int `json:"proposal_ref"` + Action string `json:"action"` + Correction *StructuredCorrectionProposal `json:"correction"` +} + +// DecodeRepairResponse checks correspondence after structural schema validation. +func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) { + var response RepairResponse + if err := json.Unmarshal(content, &response); err != nil { + return RepairResponse{}, invalidRepairResponse(err) + } + if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) { + return RepairResponse{}, invalidRepairResponse(nil) + } + return response, nil +} + +func invalidRepairResponse(cause error) error { + return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "correction repair response is unusable", Cause: cause} +} diff --git a/internal/framework/proposal_generation/repair_test.go b/internal/framework/proposal_generation/repair_test.go new file mode 100644 index 0000000..f88af7d --- /dev/null +++ b/internal/framework/proposal_generation/repair_test.go @@ -0,0 +1,35 @@ +package proposal_generation + +import ( + "errors" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" +) + +func TestDecodeRepairResponse(t *testing.T) { + for _, tc := range []struct { + name, content string + valid bool + }{ + {"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true}, + {"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true}, + {"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false}, + {"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false}, + {"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := DecodeRepairResponse([]byte(tc.content), 2) + if tc.valid && err != nil { + t.Fatal(err) + } + if !tc.valid { + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable { + t.Fatalf("failure = %v", err) + } + } + }) + } +} diff --git a/internal/framework/responseschema/assets/correction_repair.v1.json b/internal/framework/responseschema/assets/correction_repair.v1.json new file mode 100644 index 0000000..f59786d --- /dev/null +++ b/internal/framework/responseschema/assets/correction_repair.v1.json @@ -0,0 +1,57 @@ +{ + "type": "object", + "additionalProperties": false, + "required": [ + "proposal_ref", + "action", + "correction" + ], + "properties": { + "proposal_ref": { + "type": "integer", + "minimum": 1 + }, + "action": { + "type": "string", + "enum": [ + "revise", + "withdraw" + ] + }, + "correction": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "original_text", + "corrected_text", + "confidence" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "original_text": { + "type": "string", + "minLength": 1 + }, + "corrected_text": { + "type": "string" + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + { + "type": "null" + } + ] + } + } +} diff --git a/internal/framework/responseschema/registry.go b/internal/framework/responseschema/registry.go index 6922998..87aaf00 100644 --- a/internal/framework/responseschema/registry.go +++ b/internal/framework/responseschema/registry.go @@ -16,9 +16,11 @@ type Key string const ( CorrectionSetKey Key = "correction_set" + CorrectionRepairKey Key = "correction_repair" ValidatorDecisionSetKey Key = "validator_decision_set" AssetRoot = "assets" correctionSetSchemaID = "audita.correction_set" + correctionRepairSchemaID = "audita.correction_repair" validatorDecisionSchemaID = "audita.validator_decision_set" schemaVersionV1 = "v1" ) @@ -33,6 +35,8 @@ func AssetPath(key Key) (string, bool) { switch key { case CorrectionSetKey: return "correction_set.v1.json", true + case CorrectionRepairKey: + return "correction_repair.v1.json", true case ValidatorDecisionSetKey: return "validator_decision_set.v1.json", true default: @@ -59,6 +63,7 @@ func (s Schema) DiagnosticsMap() map[string]any { } var registry = map[Key]Schema{ + CorrectionRepairKey: mustBuildSchemaFromAsset(correctionRepairSchemaID, schemaVersionV1, "audita_correction_repair_v1", "assets/correction_repair.v1.json"), CorrectionSetKey: mustBuildSchemaFromAsset( correctionSetSchemaID, schemaVersionV1, diff --git a/internal/framework/responseschema/registry_test.go b/internal/framework/responseschema/registry_test.go index 99f21f5..bb5a5d3 100644 --- a/internal/framework/responseschema/registry_test.go +++ b/internal/framework/responseschema/registry_test.go @@ -10,7 +10,7 @@ import ( ) func TestPromptKitSchemaPathsUseCanonicalAssets(t *testing.T) { - for _, key := range []Key{CorrectionSetKey, ValidatorDecisionSetKey} { + for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} { path, ok := AssetPath(key) if !ok { t.Fatalf("missing path for %q", key) @@ -81,7 +81,7 @@ func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) { } func TestSchemaHashesMatchRegisteredJSON(t *testing.T) { - for _, key := range []Key{CorrectionSetKey, ValidatorDecisionSetKey} { + for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} { schema, ok := Lookup(key) if !ok { t.Fatalf("missing schema %q", key)