diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e04f4a5..f488430 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -208,6 +208,11 @@ final validation without adding transitional code or an intermediate gate. ## Stage 7: Use Prepared Plans For Both Smoke Output Checks +**Completed:** Smoke inference prepares separate JSON and schema plans and +checks syntax before schema matching. Go 1.27.3 `go test ./internal/usecase +-run 'Smoke|Diagnostic'` passed, including malformed, scalar, matching, +nonmatching, cancellation, and dispatch cases. + **Finding:** 4. **Files:** `internal/usecase/smoke_inference_case.go` and diagnostic tests. 1. Replace the smoke case's `jsonValidator validate.Validator` with a diff --git a/internal/usecase/smoke_inference_case.go b/internal/usecase/smoke_inference_case.go index 6936e5b..d3aabda 100644 --- a/internal/usecase/smoke_inference_case.go +++ b/internal/usecase/smoke_inference_case.go @@ -25,7 +25,7 @@ type smokeInferenceCase struct { kind domain.SmokeTestKind request domain.GenerateRequest expectedContent string - jsonValidator validate.Validator + jsonValidation validate.PreparedValidation schemaValidation validate.PreparedValidation } @@ -52,6 +52,12 @@ func prepareSmokeInferenceCase( }, nil case domain.SmokeTestStructuredOutput: validator := validate.NewFSValidator(smokeSchemaFS, ".") + jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{ + Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, + }) + if err != nil { + return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err) + } plan, err := validator.PrepareValidation(ctx, domain.OutputContract{ Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, @@ -64,7 +70,7 @@ func prepareSmokeInferenceCase( return &smokeInferenceCase{ kind: kind, expectedContent: smokeStructuredExpectedContent, - jsonValidator: validate.NewStandardValidator(""), + jsonValidation: jsonPlan, schemaValidation: plan, request: newGenerationRequest( domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}}, @@ -97,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene return facts, nil case domain.SmokeTestStructuredOutput: artifact := domain.Artifact{Body: []byte(response.Content)} - jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{ - Format: domain.FormatJSON, - ValidationMode: domain.ValidationJSON, - }) + jsonResult, err := c.jsonValidation.Validate(ctx, &artifact) if err != nil { return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err) } diff --git a/internal/usecase/smoke_inference_case_test.go b/internal/usecase/smoke_inference_case_test.go index 7b0bb26..b3607ce 100644 --- a/internal/usecase/smoke_inference_case_test.go +++ b/internal/usecase/smoke_inference_case_test.go @@ -2,6 +2,7 @@ package usecase import ( "context" + "errors" "reflect" "strings" "testing" @@ -102,6 +103,7 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) { }{ {name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)}, {name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, + {name: "scalar JSON misses schema", content: `42`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil}, {name: "empty content", content: "", wantSchemaMatched: nil}, @@ -125,3 +127,19 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) { t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) } } + +func TestSmokeStructuredInferenceCaseHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := prepareSmokeInferenceCase(ctx, domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0); !errors.Is(err, context.Canceled) { + t.Fatalf("prepare error = %v, want cancellation", err) + } + caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0) + if err != nil { + t.Fatal(err) + } + facts, err := caseValue.evaluate(ctx, &domain.GenerateResponse{Content: `{"status":"ok"}`}) + if !errors.Is(err, context.Canceled) || facts.ValidJSON != nil || facts.SchemaMatched != nil { + t.Fatalf("evaluation facts = %+v, error = %v", facts, err) + } +}