Keep cached scene identities aligned with the current plan
This commit is contained in:
@@ -159,6 +159,10 @@ retained.
|
||||
|
||||
## Finding 3: Rebind Cached Scene Identity To The Current Chunk Plan
|
||||
|
||||
**Status: Complete.** Scene hits require one exact current-range scene and
|
||||
reattach its current ordinal identity without model work or reassessment.
|
||||
Expansion/collapse integration regressions and scene-module tests pass.
|
||||
|
||||
Priority: P1. Relevant stages: 11, 23, 27–29, and 37.
|
||||
|
||||
### Evidence And Impact
|
||||
|
||||
124
internal/cli/scene_cache_identity_test.go
Normal file
124
internal/cli/scene_cache_identity_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type auditChangingPartitionClient struct {
|
||||
plans atomic.Int32
|
||||
descriptions atomic.Int32
|
||||
collapse bool
|
||||
}
|
||||
|
||||
func (client *auditChangingPartitionClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if productionSchemaHasProperty(request, "scenes") {
|
||||
if (client.plans.Add(1) == 1) != client.collapse {
|
||||
return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":1},{"start_unit_id":2,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil
|
||||
}
|
||||
client.descriptions.Add(1)
|
||||
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Travel","summary":"The party continues its journey."}`}, nil
|
||||
}
|
||||
|
||||
func TestSceneCacheRemainsUsableAfterPartitionRenumbering(t *testing.T) {
|
||||
t.Run("expand", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, false) })
|
||||
t.Run("collapse", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, false) })
|
||||
t.Run("expand-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, true) })
|
||||
t.Run("collapse-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, true) })
|
||||
}
|
||||
|
||||
func auditSceneCacheAfterRepartition(t *testing.T, collapse, bypass bool) {
|
||||
directory := t.TempDir()
|
||||
configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json")
|
||||
configuration := fmt.Sprintf(`version: 6
|
||||
concurrency: {workers: 1}
|
||||
output: {directory: %q}
|
||||
cache_directory: %q
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: audit
|
||||
endpoint: https://example.invalid/v1
|
||||
model: audit
|
||||
api_key_required: false
|
||||
pipelines:
|
||||
audit:
|
||||
llm_profile: audit
|
||||
input: {module: seriatim}
|
||||
chunk:
|
||||
module: dnd/scenes
|
||||
cache: refresh
|
||||
artifacts:
|
||||
scene-descriptions:
|
||||
extract: {module: dnd/scene-descriptions}
|
||||
merge: {module: appendorder}
|
||||
normalize: {module: dnd/scene-descriptions}
|
||||
`, filepath.Join(directory, "output"), filepath.Join(directory, "cache"))
|
||||
if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcript := `{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party departs."},{"id":2,"start":1,"end":2,"speaker":"GM","text":"The party travels."},{"id":3,"start":2,"end":3,"speaker":"GM","text":"The party arrives."}]}`
|
||||
if err := os.WriteFile(inputPath, []byte(transcript), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &auditChangingPartitionClient{collapse: collapse}
|
||||
for run := 1; run <= 2; run++ {
|
||||
var stdout, stderr bytes.Buffer
|
||||
args := []string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--format", "json", "--progress=false"}
|
||||
if bypass {
|
||||
args = append(args, "--cache", "bypass")
|
||||
}
|
||||
status := RunWithOptions(args, &stdout, &stderr, Options{LLMClient: client})
|
||||
if status != 0 {
|
||||
t.Fatalf("run %d: status=%d plans=%d descriptions=%d stderr=%s", run, status, client.plans.Load(), client.descriptions.Load(), stderr.String())
|
||||
}
|
||||
if run == 2 {
|
||||
var report struct {
|
||||
Directory string `json:"output_directory"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(report.Directory, "lanes", "scene-descriptions.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var artifact struct {
|
||||
Scenes []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"scenes"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantCount := 3
|
||||
if collapse {
|
||||
wantCount = 2
|
||||
}
|
||||
if len(artifact.Scenes) != wantCount {
|
||||
t.Fatalf("published scenes: %s", content)
|
||||
}
|
||||
for i, scene := range artifact.Scenes {
|
||||
if scene.ID != fmt.Sprintf("chunk-%06d", i+1) {
|
||||
t.Fatalf("published stale scene identity after repartitioning: %s", content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
wantCalls := int32(4) // The unchanged final scene is reused across runs.
|
||||
if bypass {
|
||||
wantCalls = 5
|
||||
}
|
||||
if client.descriptions.Load() != wantCalls {
|
||||
t.Fatalf("description calls=%d, want %d", client.descriptions.Load(), wantCalls)
|
||||
}
|
||||
}
|
||||
77
internal/modules/dnd/extract/scenedescriptions/cache_test.go
Normal file
77
internal/modules/dnd/extract/scenedescriptions/cache_test.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
)
|
||||
|
||||
func TestSceneCacheAdaptsOnlyUsableCurrentRange(t *testing.T) {
|
||||
for _, name := range []string{"stale identity", "wrong range", "multiple scenes", "untrimmed title"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
request := extractionRequest()
|
||||
store, err := artifactcache.NewFilesystemStore(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
codec := scenecodec.New()
|
||||
var key string
|
||||
request.Operation.Controls.Cache = contracts.CacheAuto
|
||||
request.Operation.Cache = &contracts.ProducerCache[dnd.SceneDescriptionList]{Store: store, Encode: codec.Encode, Decode: codec.Decode,
|
||||
Observe: func(_ context.Context, observation contracts.CacheObservation) error {
|
||||
key = observation.MessageContentHash
|
||||
return nil
|
||||
},
|
||||
}
|
||||
client := &fakeSceneDescriptionsLLMClient{raw: narrativeScene}
|
||||
extractor := newExtractor(t, client)
|
||||
first, err := extractor.Extract(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cached := first.Value
|
||||
cached.Scenes[0].ID = "old-chunk-identity"
|
||||
switch name {
|
||||
case "wrong range":
|
||||
cached.Scenes[0].SourceRef.EndUnitID = 2
|
||||
case "multiple scenes":
|
||||
other := cached.Scenes[0]
|
||||
other.ID = "another-old-scene"
|
||||
cached.Scenes = append(cached.Scenes, other)
|
||||
case "untrimmed title":
|
||||
cached.Scenes[0].Title = " Untrimmed "
|
||||
}
|
||||
content, err := codec.Encode(cached)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Store(ctx, contracts.CacheRefresh, key, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := extractor.Extract(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Value.Scenes) != 1 || result.Value.Scenes[0].ID != request.Chunk.ID || result.Value.Scenes[0].SourceRef != request.Chunk.Ref {
|
||||
t.Fatalf("scene does not match current chunk: %#v", result.Value)
|
||||
}
|
||||
wantCalls, wantAssessments := 2, 1
|
||||
if name == "stale identity" {
|
||||
wantCalls, wantAssessments = 1, 0
|
||||
}
|
||||
if len(client.requests) != wantCalls || assessments != wantAssessments {
|
||||
t.Fatalf("model calls=%d assessments=%d, want %d/%d", len(client.requests), assessments, wantCalls, wantAssessments)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,24 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
|
||||
chunk := source.Chunk{ID: req.Chunk.ID, Ref: req.Chunk.Ref}
|
||||
index := source.NewDocumentIndex(req.Source)
|
||||
if cache := req.Operation.Cache; cache != nil && cache.Decode != nil {
|
||||
owned := *cache
|
||||
decode := cache.Decode
|
||||
owned.Decode = func(content []byte) (dnd.SceneDescriptionList, error) {
|
||||
value, err := decode(content)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
if len(value.Scenes) != 1 || value.Scenes[0].SourceRef != chunk.Ref {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("cached scene does not describe the current chunk range")
|
||||
}
|
||||
// Chunk ordinals can change without changing the semantic prompt.
|
||||
value.Scenes = append([]dnd.SceneDescription(nil), value.Scenes...)
|
||||
value.Scenes[0].ID = chunk.ID
|
||||
return sceneFallback(&value, chunk, index)
|
||||
}
|
||||
req.Operation.Cache = &owned
|
||||
}
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
|
||||
Reference in New Issue
Block a user