Files
notarius/internal/framework/pipeline/chunk_plan_cache_test.go

426 lines
19 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// This producer uses real native preparation/continuation and returns domain
// plans; only the provider is faked by the tests.
type cachedPlanChunker struct{ engine *promptkit.Engine }
func (cachedPlanChunker) Key() string { return "cached-plan" }
func (cachedPlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c cachedPlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
references := map[string][]string{}
for name, slot := range request.References.Slots {
for _, item := range slot.Items {
references[name] = append(references[name], string(item.Content))
}
}
referenceContent, err := json.Marshal(references)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
initial := nativeProducerInitial()
initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID
initial.Inputs["source"] = promptkit.Inline(string(request.SourceInput.Content) + "\n" + string(referenceContent))
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
var plan source.ChunkPlan
err := json.Unmarshal([]byte(result.RawOutput), &plan)
return &plan, nil, err
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
return source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: request.Source.Units[0].ID, EndUnitID: request.Source.Units[len(request.Source.Units)-1].ID}}}, nil
})
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
}
func planCacheRunInput(t *testing.T, engine *promptkit.Engine, validator chunkAssessmentValidator) RunInput {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "cached-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
return cachedPlanChunker{request.Dependencies.Engine}, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "plan-policy", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.ChunkValidator, error) {
return namedPlanValidator{"plan-policy", validator}, nil
}); err != nil {
t.Fatal(err)
}
profile := typedResolutionProfile()
profile.Chunk = Binding("cached-plan")
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("plan-policy")}}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
return RunInput{Prepared: prepared, pipeline: prepared.resolved}
}
func planCacheSource(doc *source.SourceDocument) contracts.LLMInputMaterial {
var texts []string
for _, unit := range doc.Units {
texts = append(texts, unit.Text)
}
return contracts.NewLLMInputMaterial("source", "text/plain", []byte(strings.Join(texts, "\n")), "", "")
}
func planResponse(t *testing.T, plan source.ChunkPlan) *promptkit.GenerateResponse {
t.Helper()
content, err := json.Marshal(plan)
if err != nil {
t.Fatal(err)
}
return &promptkit.GenerateResponse{Content: string(content)}
}
func TestChunkPlanCacheStoresFinalPlanAndReusesWithoutAssessment(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
whole := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
final := typedTestPlan(doc)
calls, assessments := 0, 0
client := producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return planResponse(t, whole), nil
}
if calls == 2 {
if len(request.Prompt.Messages) != 4 {
t.Fatalf("correction history=%#v", request.Prompt.Messages)
}
}
return planResponse(t, final), nil
})
engine := nativeProducerEngine(t, client)
input := planCacheRunInput(t, engine, func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
if len(request.Chunks) == 1 {
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Use separate scenes."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var keys, decisions []string
input.ObserveCache = func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if origin.Stage != contracts.DiagnosticOriginStageChunk || origin.ModuleKey != "cached-plan" {
t.Fatalf("cache scope=%#v", origin)
}
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
decisions = append(decisions, observation.Decision)
return nil
}
store.beforeWrite = func(_ string, _ []byte) {
if calls != 2 || assessments != 2 {
t.Fatal("plan stored before correction acceptance")
}
}
first, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "first-session")
if err != nil || !first.accepted || len(first.chunks) != 2 || calls != 2 || assessments != 2 || store.writes != 1 {
t.Fatalf("first plan=%#v err=%v calls=%d assessments=%d", first, err, calls, assessments)
}
bytes, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
var saved source.ChunkPlan
if err != nil || json.Unmarshal(bytes, &saved) != nil || !reflect.DeepEqual(saved, *first.plan) {
t.Fatalf("cache is not the final raw plan: %s %v", bytes, err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(bytes, &fields); err != nil {
t.Fatal(err)
}
for _, field := range []string{"producer", "checkpoint", "document", "provenance", "usage"} {
if _, ok := fields[field]; ok {
t.Fatalf("cache contains reporting field %q", field)
}
}
store.beforeWrite = nil
input.pipeline.Chunk.LLMProfile = "chosen"
input.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
second, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "changed-session")
if err != nil || !second.outcome.ReuseEligible || len(second.chunks) != 2 || calls != 2 || assessments != 2 || len(keys) != 2 || keys[0] != keys[1] {
t.Fatalf("settings-only reuse=%#v %v calls=%d assessments=%d keys=%v", second, err, calls, assessments, keys)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions=%v", decisions)
}
// Materialization is against the current source, not cached transcript bytes.
changed := *doc
changed.ID = "current-source"
changed.Units = append([]source.SourceUnit(nil), doc.Units...)
for i := range changed.Units {
changed.Units[i].Ref.SourceID = changed.ID
}
changed.Digest, _ = source.DigestDocument(&changed)
// Identity is part of ordinary plan usability if it changes the source digest.
// Supplying the matching digest lets this check focus on rematerialized refs.
saved.SourceDigest = changed.Digest
replacement, _ := json.Marshal(saved)
if err := store.inner.Store(context.Background(), contracts.CacheRefresh, keys[0], replacement); err != nil {
t.Fatal(err)
}
third, err := New().runChunkPlan(context.Background(), input, &changed, planCacheSource(doc), "session")
if err != nil || calls != 2 || assessments != 2 || third.chunks[0].Ref.SourceID != changed.ID || third.chunks[0].Units[0].Ref.SourceID != changed.ID {
t.Fatalf("rematerialization=%#v %v", third, err)
}
}
func TestChunkPlanCacheUsesMessageContentRatherThanSourceIdentity(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
client := producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
plan := typedTestPlan(doc)
if calls > 1 {
plan.Ranges = []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}
}
return planResponse(t, plan), nil
})
validator := chunkAssessmentValidator(func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input := planCacheRunInput(t, nativeProducerEngine(t, client), validator)
input.Cache = store
var keys []string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
return nil
}
run := func(wantChunks int) {
t.Helper()
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || len(result.chunks) != wantChunks {
t.Fatalf("chunks=%d err=%v", len(result.chunks), err)
}
}
run(2)
input.pipeline.ChunkReferences.ReferenceSet = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte("new scene context")}}}}}
run(1)
// Replace only the prompt text, retaining source and reference bytes.
input.Prepared.chunker = cachedPlanChunker{nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("changed prompt", []byte(`prompts:
- id: initial
default_profile: base
inputs: [{name: source, required: true}]
messages:
- {role: system, content: 'A changed scene assignment.'}
- {role: user, content: 'Source: {{input "source"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`))))}
run(1)
if calls != 3 || assessments != 3 || len(keys) != 3 || keys[0] == keys[1] || keys[1] == keys[2] {
t.Fatalf("content reuse: calls=%d assessments=%d keys=%v", calls, assessments, keys)
}
}
func TestChunkPlanCacheUnusableEntriesAreMissesNotAssistantCandidates(t *testing.T) {
for _, defect := range []string{"source digest", "source range", "malformed", "legacy envelope"} {
t.Run(defect, func(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocument()
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if len(request.Prompt.Messages) != 2 {
t.Fatal("unusable cache replayed as an assistant candidate")
}
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
bad := typedTestPlan(doc)
switch defect {
case "source digest":
bad.SourceDigest = "not-current"
case "source range":
bad.Ranges[0].EndUnitID = 999
}
content, _ := json.Marshal(bad)
if defect == "malformed" {
content = []byte("{")
}
if defect == "legacy envelope" {
content = []byte(`{"plan":{},"producer":{"key":"legacy"}}`)
}
// Simulate an existing corrupt entry without going through valid writes.
if err := os.WriteFile(filepath.Join(root, key+".json"), content, 0644); err != nil {
t.Fatal(err)
}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || !result.accepted || calls != 2 || assessments != 2 || len(result.diagnostics) != 1 {
t.Fatalf("unusable plan=%#v err=%v calls=%d assessments=%d", result, err, calls, assessments)
}
})
}
}
func TestChunkPlanCacheModesAndTerminalExclusions(t *testing.T) {
for _, test := range []struct {
name string
cache contracts.CacheMode
mode contracts.ExecutionMode
reject, ineligible bool
reads, writes int
fallback bool
}{
{"auto", contracts.CacheAuto, contracts.ModeStrict, false, false, 1, 1, false},
{"bypass", contracts.CacheBypass, contracts.ModeStrict, false, false, 0, 0, false},
{"refresh", contracts.CacheRefresh, contracts.ModeStrict, false, false, 0, 1, false},
{"ineligible", contracts.CacheAuto, contracts.ModeStrict, false, true, 0, 0, false},
{"strict rejection", contracts.CacheAuto, contracts.ModeStrict, true, false, 1, 0, false},
{"permissive fallback", contracts.CacheAuto, contracts.ModePermissive, true, false, 1, 0, true},
} {
t.Run(test.name, func(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !test.reject, ReasonCode: "policy", CorrectionGuidance: "Revise the scene partition."}, nil
})
input.Cache = store
input.reuseIneligible = test.ineligible
input.pipeline.ChunkControls = contracts.ProducerControls{Cache: test.cache, Mode: test.mode, MaxAdditionalTurns: 0}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (test.reject && !test.fallback) || calls != 1 || assessments != 1 || store.reads != test.reads || store.writes != test.writes {
t.Fatalf("result=%#v err=%v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
if test.fallback && (result.outcome.Disposition != contracts.ProducerFallback || result.outcome.ReuseEligible || len(result.chunks) != 1 || len(result.chunks[0].Units) != 2) {
t.Fatalf("invalid whole-source fallback: %#v", result)
}
})
}
}
func TestChunkPlanFailedRefreshPreservesAcceptanceAndIgnoresLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacyNames := []string{"checkpoints/run/manifest.json", "chunk-plans/source/plan.json"}
for _, name := range legacyNames {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("unread legacy bytes"), 0644); err != nil {
t.Fatal(err)
}
}
doc := typedTestDocumentWithUnits(2)
reject := false
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !reject, ReasonCode: "partition", CorrectionGuidance: "Reconsider the partition."}, nil
})
input.Cache = store
input.pipeline.ChunkControls.MaxAdditionalTurns = 0
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
original, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil {
t.Fatal(err)
}
reject = true
input.pipeline.ChunkControls.Cache = contracts.CacheRefresh
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
input.pipeline.ChunkControls.Mode = mode
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (mode == contracts.ModeStrict) {
t.Fatalf("refresh mode=%s result=%#v error=%v", mode, result, err)
}
if mode == contracts.ModePermissive && (result.outcome.ReuseEligible || result.outcome.Disposition != contracts.ProducerFallback) {
t.Fatalf("fallback=%#v", result.outcome)
}
current, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil || string(current) != string(original) {
t.Fatal("failed refresh changed old acceptance")
}
}
input.pipeline.ChunkControls.Cache = contracts.CacheAuto
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || result.outcome.Disposition != contracts.ProducerAccepted || calls != 3 || assessments != 3 || store.reads != 2 || store.writes != 1 {
t.Fatalf("old acceptance not reused: %#v %v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
for _, name := range legacyNames {
content, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(content) != "unread legacy bytes" {
t.Fatalf("legacy record modified: %s %q %v", name, content, err)
}
}
}
func TestDeterministicChunkingIgnoresCacheAndLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacy := map[string][]byte{"checkpoints/identity/manifest.json": []byte("old checkpoint"), "chunk-plans/source.json": []byte("old plan")}
for name, content := range legacy {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, content, 0644); err != nil {
t.Fatal(err)
}
}
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, ObserveCache: func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error {
return fmt.Errorf("deterministic chunker prepared a synthetic request")
}}
for range 2 {
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatal(err)
}
}
if store.reads != 0 || store.writes != 0 {
t.Fatalf("deterministic cache I/O=%d/%d", store.reads, store.writes)
}
for name, want := range legacy {
got, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(got) != string(want) {
t.Fatalf("legacy record modified: %s %q %v", name, got, err)
}
}
}