Cache final typed producer artifacts under the initial native request hash

This commit is contained in:
2026-11-22 02:10:05 +00:00
parent 3ef55b1cf0
commit 36906cb3d1
15 changed files with 878 additions and 15 deletions

View File

@@ -30,6 +30,27 @@ type ArtifactCache interface {
Store(context.Context, CacheMode, string, []byte) error
}
// CacheObservation is transient operation evidence, not a persisted record or
// public report format. Prepared details are available even when lookup hits
// and native execution (and its OnPrepared callback) never happens.
type CacheObservation struct {
Decision string
MessageContentHash string
Prepared *promptkit.PreparedRun
Err error
ChunkID string
ChunkIndex *int
}
// ProducerCache binds the expected artifact codec and ordinary source usability
// to a store. Decode must not invoke the configured semantic validator chain.
type ProducerCache[T any] struct {
Store ArtifactCache
Encode func(T) ([]byte, error)
Decode func([]byte) (T, error)
Observe func(context.Context, CacheObservation) error
}
// ProducerControls is resolved before execution; modules do not inherit settings.
type ProducerControls struct {
Mode ExecutionMode `json:"mode"`
@@ -78,6 +99,7 @@ type ProducerOperation[T any] struct {
Assess CandidateAssessor[T] `json:"-"`
Execution ExecutionOptions
ReuseEligible bool
Cache *ProducerCache[T] `json:"-"`
}
type ProducerDisposition string

View File

@@ -52,13 +52,14 @@ type preparedLaneExecutor struct {
}
type preparedTypedLane struct {
extractor any
merger any
normalizer any
extract typedExtractOperation
merge typedMergeOperation
normalize typedNormalizeOperation
codec artifactCodecEntry
extractor any
merger any
normalizer any
extract typedExtractOperation
merge typedMergeOperation
normalize typedNormalizeOperation
codec artifactCodecEntry
cacheEvidence func(any) ([]source.SourceRef, error)
}
type preparedEvidencePlan struct {
@@ -228,6 +229,13 @@ func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registrie
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
}
executor.typed = &preparedTypedLane{extractor: module, extract: extractEntry.extract, codec: codec}
if registry := registries.ArtifactEvidence; registry != nil {
if evidence, ok := registry.entries[lane.ArtifactKind]; ok {
// Projectors retain their own exact-type check. A mismatch is an
// unusable cache entry, not a reason to inspect domain JSON.
executor.typed.cacheEvidence = evidence.project
}
}
executor.extractValidators, err = prepareValidatorChain(pipeline, registries, deps, StageExtract, lane.ID, lane.Extract.Module, lane.ExtractReferences.ReferenceSet)
if err != nil {

View File

@@ -85,7 +85,47 @@ func RunProducer[T any](
request := producerRequest(initial, operation.Execution, true)
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
result, err := engine.Run(ctx, request)
var result *promptkit.RunResult
var err error
var cacheKey string
cache := operation.Cache
if cache != nil && cache.Store != nil && operation.ReuseEligible && operation.Controls.Cache != contracts.CacheBypass {
if cache.Encode == nil || cache.Decode == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer cache codec must be supplied")
}
if mode := operation.Controls.Cache; mode != "" && mode != contracts.CacheAuto && mode != contracts.CacheRefresh {
return zero, contracts.ProducerOutcome{}, fmt.Errorf("%w: unknown cache mode %q", promptkit.ErrInvalidRequest, mode)
}
var prepared *promptkit.PreparedExecution
prepared, err = engine.PrepareExecution(ctx, request)
if err == nil {
defer prepared.Discard()
details := prepared.Details()
cacheKey = details.MessageContentHash
if captureErr := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "prepared", MessageContentHash: cacheKey, Prepared: &details}); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
reused, lookupErr := lookupProducerCache(ctx, cache, operation.Controls.Cache, cacheKey)
if lookupErr != nil {
return zero, contracts.ProducerOutcome{}, lookupErr
}
if reused != nil {
return *reused, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}, nil
}
result, err = engine.RunPrepared(ctx, prepared)
}
} else {
if cache != nil {
decision := "bypass"
if !operation.ReuseEligible {
decision = "ineligible"
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: decision}); err != nil {
return zero, contracts.ProducerOutcome{}, err
}
}
result, err = engine.Run(ctx, request)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
@@ -149,6 +189,11 @@ func RunProducer[T any](
}
}
if accepted {
if cacheKey != "" {
if err := storeProducerCache(ctx, cache, operation.Controls.Cache, cacheKey, *candidate); err != nil {
return zero, contracts.ProducerOutcome{}, err
}
}
return *candidate, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: operation.ReuseEligible}, nil
}
var construct func(context.Context) (T, error)

View File

@@ -0,0 +1,67 @@
package pipeline
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func observeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCache[T], observation contracts.CacheObservation) error {
if err := ctx.Err(); err != nil {
return err
}
if cache != nil && cache.Observe != nil {
if err := cache.Observe(ctx, observation); err != nil {
return err
}
}
return ctx.Err()
}
func lookupProducerCache[T any](ctx context.Context, cache *contracts.ProducerCache[T], mode contracts.CacheMode, hash string) (*T, error) {
if mode == contracts.CacheRefresh {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "refresh", MessageContentHash: hash})
}
content, err := cache.Store.Lookup(ctx, mode, hash)
if canceled := ctx.Err(); canceled != nil {
return nil, canceled
}
if err != nil {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "lookup_failed", MessageContentHash: hash, Err: err})
}
if content == nil {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "miss", MessageContentHash: hash})
}
value, err := cache.Decode(content)
if canceled := ctx.Err(); canceled != nil {
return nil, canceled
}
if err != nil {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err})
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash}); err != nil {
return nil, err
}
return &value, nil
}
func storeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCache[T], mode contracts.CacheMode, hash string, value T) error {
if err := ctx.Err(); err != nil {
return err
}
content, err := cache.Encode(value)
if err != nil {
// Artifact encoding is an ordinary application boundary, not a cache I/O
// failure. Do not hide an unusable producer value as a cache warning.
return fmt.Errorf("encode accepted artifact: %w", err)
}
if err := ctx.Err(); err != nil {
return err
}
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash}
if err := cache.Store.Store(ctx, mode, hash, content); err != nil {
observation.Decision, observation.Err = "store_failed", err
}
return observeProducerCache(ctx, cache, observation)
}

View File

@@ -0,0 +1,344 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type producerCacheStore struct {
inner contracts.ArtifactCache
reads, writes int
readErr, writeErr error
beforeWrite func(string, []byte)
}
func (store *producerCacheStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) {
store.reads++
if store.readErr != nil {
return nil, store.readErr
}
return store.inner.Lookup(ctx, mode, hash)
}
func (store *producerCacheStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error {
store.writes++
if store.beforeWrite != nil {
store.beforeWrite(hash, content)
}
if store.writeErr != nil {
return store.writeErr
}
return store.inner.Store(ctx, mode, hash, content)
}
func numberCacheOperation(store contracts.ArtifactCache) contracts.ProducerOperation[int] {
return contracts.ProducerOperation[int]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheAuto, MaxAdditionalTurns: 1},
ReuseEligible: true, Assess: acceptedProducerAssessment,
Cache: &contracts.ProducerCache[int]{
Store: store, Encode: func(value int) ([]byte, error) { return json.Marshal(value) },
Decode: func(content []byte) (int, error) {
var value int
err := json.Unmarshal(content, &value)
return value, err
},
},
}
}
func filesystemProducerCache(t *testing.T) (string, *producerCacheStore) {
t.Helper()
root := t.TempDir()
store, err := artifactcache.NewFilesystemStore(root)
if err != nil {
t.Fatal(err)
}
return root, &producerCacheStore{inner: store}
}
func TestProducerCachesAppliedCorrectionUnderInitialKeyAndReusesWithoutExecution(t *testing.T) {
root, store := filesystemProducerCache(t)
calls, assessments, nativePrepared, events := 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
value := 2
if calls == 0 {
value = 0
}
calls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
operation := numberCacheOperation(store)
operation.Execution = contracts.ExecutionOptions{
OnPrepared: func(context.Context, promptkit.PreparedRun) { nativePrepared++ },
Observer: func(promptkit.ExecutionEvent) { events++ },
}
var keys []string
var decisions []string
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
decisions = append(decisions, observation.Decision)
if observation.Prepared != nil {
if observation.Prepared.MessageContentHash == "" || len(observation.Prepared.Messages) == 0 {
t.Fatal("lost prepared details")
}
keys = append(keys, observation.MessageContentHash)
}
return nil
}
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
if value == 20 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
}
// The response is a proposal; the cache must contain the applied typed value,
// not the private response or an earlier candidate.
interpretApplied := func(ctx context.Context, response *promptkit.RunResult) (*int, []string, error) {
value, guidance, err := interpretProducerNumber(ctx, response)
if value != nil {
*value *= 10
}
return value, guidance, err
}
store.beforeWrite = func(hash string, content []byte) {
if assessments != 2 || calls != 2 || string(content) != "20" || hash != keys[0] {
t.Fatalf("early or incorrect cache store: %s %q, calls=%d assessments=%d", hash, content, calls, assessments)
}
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretApplied, producerCorrection, nil)
if err != nil || value != 20 || outcome.Disposition != contracts.ProducerAccepted || store.writes != 1 || nativePrepared != 2 || events == 0 {
t.Fatalf("accepted correction = %d %#v %v; writes %d, prepared %d, events %d", value, outcome, err, store.writes, nativePrepared, events)
}
content, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
if err != nil || string(content) != "20" {
t.Fatalf("stored artifact = %q, %v", content, err)
}
priorEvents := events
store.beforeWrite = nil
initial := nativeProducerInitial()
initial.SessionID, initial.ProfileID = "another-session", "chosen"
operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
t.Fatal("cache hit reassessed")
return contracts.CandidateAssessment{}, nil
}
value, outcome, err = RunProducer(context.Background(), engine, operation, initial,
func(context.Context, *promptkit.RunResult) (*int, []string, error) {
t.Fatal("cache hit interpreted a model result")
return nil, nil, nil
}, producerCorrection, nil)
if err != nil || value != 20 || !outcome.ReuseEligible || calls != 2 || assessments != 2 || nativePrepared != 2 || events != priorEvents || store.writes != 1 || len(keys) != 2 || keys[0] != keys[1] {
t.Fatalf("settings/session reuse = %d %#v %v; calls=%d assessments=%d native-prepared=%d events=%d keys=%v", value, outcome, err, calls, assessments, nativePrepared, events, keys)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions = %v", decisions)
}
operation.Assess = acceptedProducerAssessment
initial.Inputs["source"] = promptkit.Inline("changed source content")
value, _, err = RunProducer(context.Background(), engine, operation, initial, interpretApplied, producerCorrection, nil)
if err != nil || value != 20 || calls != 3 || len(keys) != 3 || keys[0] == keys[2] || store.writes != 2 {
t.Fatalf("content change did not miss: value %d error %v calls %d keys %v", value, err, calls, keys)
}
}
func TestProducerExecutesFrozenPreparedHandleAfterLookup(t *testing.T) {
_, store := filesystemProducerCache(t)
initial := nativeProducerInitial()
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if len(request.Prompt.Messages) != 2 || request.Prompt.Messages[1].Content != "Source: fixed source" {
t.Fatalf("request was prepared again after cache lookup: %#v", request.Prompt.Messages)
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
var hash string
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
hash = observation.MessageContentHash
initial.Inputs["source"] = promptkit.Inline("mutation after preparation")
observation.Prepared.Messages[1].Content = "mutated detached details"
observation.Prepared.MessageContentHash = "not-the-key"
}
return nil
}
store.beforeWrite = func(key string, _ []byte) {
if key != hash {
t.Fatalf("mutable snapshot changed key: %s", key)
}
}
if value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, nil, nil); err != nil || value != 1 {
t.Fatalf("prepared execution = %d, %v", value, err)
}
}
func TestProducerCacheSkipsIneligibleBypassedAndNonacceptedWrites(t *testing.T) {
for _, test := range []struct {
name string
mode contracts.CacheMode
eligible, reject, permissive, cancel, providerFailure, structuralFailure bool
wantReads, wantWrites int
}{
{name: "bypass", mode: contracts.CacheBypass, eligible: true},
{name: "ineligible", mode: contracts.CacheAuto},
{name: "refresh", mode: contracts.CacheRefresh, eligible: true, wantWrites: 1},
{name: "strict rejection", mode: contracts.CacheAuto, eligible: true, reject: true, wantReads: 1},
{name: "permissive fallback", mode: contracts.CacheAuto, eligible: true, reject: true, permissive: true, wantReads: 1},
{name: "cancellation", mode: contracts.CacheAuto, eligible: true, cancel: true, wantReads: 1},
{name: "provider failure", mode: contracts.CacheAuto, eligible: true, providerFailure: true, wantReads: 1},
{name: "structural failure", mode: contracts.CacheAuto, eligible: true, structuralFailure: true, wantReads: 1},
} {
t.Run(test.name, func(t *testing.T) {
_, store := filesystemProducerCache(t)
providerFailure := errors.New("offline provider failure")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if test.providerFailure {
return nil, providerFailure
}
if test.structuralFailure {
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
operation := numberCacheOperation(store)
operation.Controls.Cache, operation.ReuseEligible = test.mode, test.eligible
operation.Controls.MaxAdditionalTurns = 0
if test.permissive {
operation.Controls.Mode = contracts.ModePermissive
}
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
if test.cancel {
cancel()
}
if test.reject {
return rejectedProducerAssessment(ctx, value)
}
return acceptedProducerAssessment(ctx, value)
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection,
func(context.Context, *int) (int, error) { return -1, nil })
switch {
case test.cancel:
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
t.Fatalf("canceled=%#v %v", outcome, err)
}
case test.providerFailure:
if !errors.Is(err, providerFailure) {
t.Fatalf("provider failure=%v", err)
}
case test.permissive:
if err != nil || outcome.Disposition != contracts.ProducerFallback {
t.Fatalf("fallback=%#v %v", outcome, err)
}
case test.reject || test.structuralFailure:
if err == nil {
t.Fatal("rejection accepted")
}
default:
if err != nil || outcome.Disposition != contracts.ProducerAccepted {
t.Fatalf("acceptance=%#v %v", outcome, err)
}
}
if store.reads != test.wantReads || store.writes != test.wantWrites {
t.Fatalf("cache reads/writes=%d/%d; want %d/%d", store.reads, store.writes, test.wantReads, test.wantWrites)
}
})
}
}
func TestProducerCacheMissesOnUnusableArtifactAndTreatsIOAsDiagnostic(t *testing.T) {
for _, scenario := range []string{"unusable", "io failure"} {
t.Run(scenario, func(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
var diagnostics []contracts.CacheObservation
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
if observation.Prepared != nil && scenario == "unusable" {
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte(`{"wrong":"artifact"}`))
}
if observation.Err != nil {
diagnostics = append(diagnostics, observation)
}
return nil
}
if scenario == "io failure" {
store.readErr = errors.New("unreadable cache")
store.writeErr = errors.New("unwritable cache")
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 1 || len(diagnostics) == 0 {
t.Fatalf("cache diagnostic changed successful work: %d %#v %v, calls %d, diagnostics %v", value, outcome, err, calls, diagnostics)
}
})
}
}
func TestProducerCacheDoesNotHideEncodingOrCaptureFailureWithFallback(t *testing.T) {
for _, boundary := range []string{"encode", "capture"} {
t.Run(boundary, func(t *testing.T) {
_, store := filesystemProducerCache(t)
failure := errors.New(boundary + " failed")
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
operation.Controls.Mode = contracts.ModePermissive
if boundary == "encode" {
operation.Cache.Encode = func(int) ([]byte, error) { return nil, failure }
} else {
operation.Cache.Observe = func(context.Context, contracts.CacheObservation) error { return failure }
}
fallbacks := 0
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, failure) || fallbacks != 0 || store.writes != 0 || outcome.Disposition != "" {
t.Fatalf("%s failure=%#v %v, fallback=%d", boundary, outcome, err, fallbacks)
}
if boundary == "capture" && calls != 0 {
t.Fatal("execution started despite failed prepared capture")
}
})
}
}
func TestProducerCacheFailedRefreshPreservesPreviousArtifact(t *testing.T) {
root, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, errors.New("refresh failed")
}))
prepared, err := engine.PrepareExecution(context.Background(), nativeProducerInitial())
if err != nil {
t.Fatal(err)
}
key := prepared.Details().MessageContentHash
prepared.Discard()
if err := store.inner.Store(context.Background(), contracts.CacheAuto, key, []byte("7")); err != nil {
t.Fatal(err)
}
operation := numberCacheOperation(store)
operation.Controls.Cache = contracts.CacheRefresh
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err == nil {
t.Fatal("refresh failure accepted")
}
content, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil || strings.TrimSpace(string(content)) != "7" || store.reads != 0 || store.writes != 0 {
t.Fatalf("refresh destroyed prior acceptance: %q %v, reads/writes %d/%d", content, err, store.reads, store.writes)
}
}

View File

@@ -41,6 +41,10 @@ func New() *Runner {
}
type RunInput struct {
Cache contracts.ArtifactCache
// ObserveCache receives scoped transient evidence, including prepared details
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string

View File

@@ -0,0 +1,64 @@
package pipeline
import (
"context"
"fmt"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func artifactProducerCache(input RunInput, typed *preparedTypedLane, class contracts.ExecutionClass, origin contracts.DiagnosticOrigin, doc *source.SourceDocument, chunk *source.Chunk, warnings *[]contracts.ProducerDiagnostic) *contracts.ProducerCache[any] {
if input.Cache == nil || class != contracts.ExecutionClassLLMBacked {
return nil
}
index := source.NewDocumentIndex(doc)
return &contracts.ProducerCache[any]{
Store: input.Cache,
Encode: typed.codec.encode,
Decode: func(content []byte) (any, error) {
value, err := typed.codec.decode(content)
if err != nil {
return nil, err
}
if typed.cacheEvidence != nil {
refs, err := typed.cacheEvidence(value)
if err != nil {
return nil, err
}
for _, ref := range refs {
if err := index.ValidateRef(ref); err != nil {
return nil, fmt.Errorf("cached artifact source reference: %w", err)
}
}
}
return value, nil
},
Observe: func(ctx context.Context, observation contracts.CacheObservation) error {
if chunk != nil {
chunkIndex := chunk.Index
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
}
if observation.Err != nil {
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
if len(message) > contracts.MaxDiagnosticMessageBytes {
message = message[:contracts.MaxDiagnosticMessageBytes]
for !utf8.ValidString(message) {
message = message[:len(message)-1]
}
}
*warnings = append(*warnings, contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: message}},
})
}
if input.ObserveCache != nil {
return input.ObserveCache(ctx, origin, observation)
}
return nil
},
}
}

View File

@@ -0,0 +1,253 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
func TestRunnerReusesAcceptedPredecessorAfterDownstreamFailure(t *testing.T) {
_, store := filesystemProducerCache(t)
probe := newRunnerProbe()
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "cached-producer", probe: probe, native: true})
consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe})
profile := typedResolutionProfile()
profile.Artifacts = nil
profile.Steps = []PipelineStepProfile{
{ID: "produce", Artifacts: map[string]ArtifactLaneProfile{"producer": producer}},
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer}},
}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
failure := errors.New("downstream application failure")
first := true
consumerOperation := prepared.Steps[1].lanes[0].typed.extract
prepared.Steps[1].lanes[0].typed.extract = func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
if first {
return erasedTypedResult{}, failure
}
return consumerOperation(ctx, implementation, request)
}
var observations []contracts.CacheObservation
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Cache: store,
ObserveCache: func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if origin.ModuleKey != "cached-producer" || origin.Stage != contracts.DiagnosticOriginStageExtract || observation.ChunkID == "" || observation.ChunkIndex == nil {
t.Fatalf("lost cache scope: %#v %#v", origin, observation)
}
observations = append(observations, observation)
return nil
},
}
if _, err := New().Run(context.Background(), input); !errors.Is(err, failure) {
t.Fatalf("downstream failure=%v", err)
}
if calls != 1 || store.reads != 1 || store.writes != 1 {
t.Fatalf("predecessor not stored before downstream failure: calls %d reads/writes %d/%d", calls, store.reads, store.writes)
}
first = false
output, err := New().Run(context.Background(), input)
if err != nil || len(output.NormalizeOutputs) != 2 || calls != 1 || store.reads != 2 || store.writes != 1 || output.FallbackCount != 0 {
t.Fatalf("rerun=%v outputs=%d calls=%d cache=%d/%d", err, len(output.NormalizeOutputs), calls, store.reads, store.writes)
}
var decisions []string
for _, observation := range observations {
decisions = append(decisions, observation.Decision)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions=%v", decisions)
}
// The deterministic merge/normalize work runs normally on both invocations;
// the reused producer's semantic assessor is not repeated.
if probe.calls["cached-producer/merge"] != 2 || probe.calls["cached-producer/normalize"] != 2 {
t.Fatalf("deterministic work was incorrectly cached: %v", probe.calls)
}
}
func TestRunnerCacheWarningsDoNotFailSuccessfulPublication(t *testing.T) {
_, store := filesystemProducerCache(t)
store.readErr = errors.New("cache read unavailable")
store.writeErr = errors.New(strings.Repeat("cache write unavailable ", 400))
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), Cache: store, Workers: 1})
if err != nil || len(output.NormalizeOutputs) != 1 || output.FallbackCount != 0 {
t.Fatalf("cache warnings failed successful work: %v, outputs=%d", err, len(output.NormalizeOutputs))
}
projection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
if err != nil || projection.WarningOccurrenceCount != 2 {
t.Fatalf("cache diagnostics=%#v %v", projection, err)
}
for _, warning := range projection.Warnings {
if warning.Origin.ModuleKey != "native" || warning.Origin.LaneID != "notes" || warning.Disposition != contracts.DiagnosticDispositionWarning {
t.Fatalf("unscoped cache warning=%#v", warning)
}
}
}
func TestTypedCacheDecodeUsesCodecAndCurrentSourceWithoutSemanticAssessment(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocument()
codec := notesCodec()
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, codec); err != nil {
t.Fatal(err)
}
entry, _, err := registry.entry("test/notes")
if err != nil {
t.Fatal(err)
}
evidence := NewArtifactEvidenceRegistry()
if err := RegisterArtifactEvidence(evidence, "test/notes", func(value codecNotes) []source.SourceRef {
if value.Details == nil {
return nil
}
return []source.SourceRef{{SourceID: value.Details.Name, StartUnitID: 1, EndUnitID: 1}}
}); err != nil {
t.Fatal(err)
}
projector, _, err := evidence.entry("test/notes")
if err != nil {
t.Fatal(err)
}
typed := &preparedTypedLane{codec: entry, cacheEvidence: projector.project}
var warnings []contracts.ProducerDiagnostic
cache := artifactProducerCache(RunInput{Cache: store}, typed, contracts.ExecutionClassLLMBacked,
contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, ModuleKey: "native"}, doc, nil, &warnings)
specialized := specializeProducerOperation[codecNotes](contracts.ProducerOperation[any]{Cache: cache}).Cache
for _, test := range []struct {
name string
content []byte
valid bool
}{
{"current source", []byte(`{"items":["accepted"],"details":{"name":"source"}}`), true},
{"stale source", []byte(`{"items":["accepted"],"details":{"name":"old-source"}}`), false},
{"unusable codec", []byte(`{"items":"not a collection"}`), false},
} {
t.Run(test.name, func(t *testing.T) {
value, err := specialized.Decode(test.content)
if (err == nil) != test.valid {
t.Fatalf("decoded=%#v %v", value, err)
}
})
}
// A codec mismatch stays a typed error at the registry boundary.
wrong := specializeProducerOperation[codecScore](contracts.ProducerOperation[any]{Cache: cache}).Cache
if _, err := wrong.Decode([]byte(`{"items":[]}`)); err == nil {
t.Fatal("cache decode crossed an incompatible Go type")
}
}
func TestProducerCacheCancellationDuringHitCannotDeliverAcceptance(t *testing.T) {
_, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
t.Fatal("cache hit made a provider call")
return nil, nil
}))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
operation := numberCacheOperation(store)
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte("1"))
}
if observation.Decision == "hit" {
cancel()
}
return nil
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
t.Fatalf("canceled hit delivered acceptance: %#v %v", outcome, err)
}
}
func TestRunnerPreparesCacheEvidenceWithoutEnablingEvidenceOutput(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
catalog.ArtifactEvidence = NewArtifactEvidenceRegistry()
if err := RegisterArtifactEvidence(catalog.ArtifactEvidence, "test/notes", func(value codecNotes) []source.SourceRef {
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}
}); err != nil {
t.Fatal(err)
}
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
for _, lane := range prepared.Steps[0].lanes {
if lane.resolved.ArtifactKind == "test/notes" {
if lane.typed.cacheEvidence == nil {
t.Fatal("source usability depended on evidence-context publication")
}
content, _ := json.Marshal(codecNotes{Items: []string{}})
_, store := filesystemProducerCache(t)
var warnings []contracts.ProducerDiagnostic
cache := artifactProducerCache(RunInput{Cache: store}, lane.typed, contracts.ExecutionClassLLMBacked, contracts.DiagnosticOrigin{}, typedTestDocument(), nil, &warnings)
if _, err := cache.Decode(content); err != nil {
t.Fatal(err)
}
}
}
}
func TestRunnerCachesAppliedMergeAndNormalizationArtifacts(t *testing.T) {
for _, role := range []ModuleStage{StageMerge, StageNormalize} {
t.Run(string(role), func(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
prepared := preparedConcurrentPipeline(t, 1)
lane := &prepared.Steps[0].lanes[0]
produce := func(ctx context.Context, operation contracts.ProducerOperation[any]) (erasedTypedResult, error) {
value, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
func(context.Context, *promptkit.RunResult) (*any, []string, error) {
var applied any = codecNotes{Items: []string{"applied"}}
return &applied, nil, nil
}, nil, nil)
return erasedTypedResult{Value: value, Outcome: outcome}, err
}
if role == StageMerge {
lane.resolved.MergeExecutionClass = contracts.ExecutionClassLLMBacked
lane.typed.merge = func(ctx context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
return produce(ctx, request.Operation)
}
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: request.MergeOutput.Value}, ctx.Err()
}
} else {
lane.resolved.NormalizeExecutionClass = contracts.ExecutionClassLLMBacked
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return produce(ctx, request.Operation)
}
}
for range 2 {
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: store, RawInput: []byte("input"), Workers: 1})
if err != nil || len(output.NormalizeOutputs) != 2 {
t.Fatalf("cached %s execution: %v, outputs %d", role, err, len(output.NormalizeOutputs))
}
value, err := notesCodec().Decode(output.NormalizeOutputs[0].Artifact.Content)
if err != nil || !reflect.DeepEqual(value.Items, []string{"applied"}) {
t.Fatalf("applied artifact changed on reuse: %#v, %v", value, err)
}
}
if calls != 1 || store.reads != 2 || store.writes != 1 {
t.Fatalf("%s cache calls: provider %d, reads/writes %d/%d", role, calls, store.reads, store.writes)
}
})
}
}

View File

@@ -75,7 +75,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
}
result.plan, result.chunks, result.accepted = &plan, chunks, true
result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, nil, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.setValidation(nil, err)
return result, err
}
@@ -134,7 +134,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if outcome.Disposition == contracts.ProducerFallback {
result.summary.ValidationStatus = "incomplete"
}
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, nil, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
if err != nil {
return result, err
}

View File

@@ -457,6 +457,8 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
return assessment, err
},
}
var cacheDiagnostics []contracts.ProducerDiagnostic
operation.Cache = artifactProducerCache(input, typed, lane.ExtractExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, doc, &chunk, &cacheDiagnostics)
extracted, err := typed.extract(ctx, typed.extractor, contracts.TypedExtractionRequest[any]{Operation: operation, Source: doc, Chunk: &chunk, SourceInput: target.sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Extract.LLMProfile, Metadata: metadata})
if err != nil {
result.err = fmt.Errorf("extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
@@ -473,7 +475,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
return result
}
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = chunk.ID, chunk.Index, chunk.Ref
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
return result

View File

@@ -38,11 +38,16 @@ func completeProducerOperation[T any](ctx context.Context, class contracts.Execu
return value, outcome, nil
}
func operationDiagnosticGroups(diagnostics []contracts.ProducerDiagnostic, assessment contracts.CandidateAssessment, outcome contracts.ProducerOutcome, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
func operationDiagnosticGroups(diagnostics, cacheDiagnostics []contracts.ProducerDiagnostic, assessment contracts.CandidateAssessment, outcome contracts.ProducerOutcome, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
groups, err := promoteProducerDiagnostics(diagnostics, origin, chunk)
if err != nil {
return nil, err
}
cacheGroups, err := promoteProducerDiagnostics(cacheDiagnostics, origin, chunk)
if err != nil {
return nil, err
}
groups = append(groups, cacheGroups...)
if outcome.Disposition == contracts.ProducerFallback {
warning := contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback,

View File

@@ -278,6 +278,8 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
return assessment, err
},
}
var cacheDiagnostics []contracts.ProducerDiagnostic
operation.Cache = artifactProducerCache(input, typed, lane.MergeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.merge(ctx, typed.merger, contracts.TypedMergeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, ExtractOutputs: inputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Merge.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
@@ -293,7 +295,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
if err != nil {
return state, fmt.Errorf("encode merge lane %q: %w", lane.ID, err)
}
diagnostics, err := operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
diagnostics, err := operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
if err != nil {
return state, err
}
@@ -362,6 +364,8 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
return assessment, err
},
}
var cacheDiagnostics []contracts.ProducerDiagnostic
operation.Cache = artifactProducerCache(input, typed, lane.NormalizeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.normalize(ctx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: merged.LaneID, MergerKey: merged.MergerKey, SourceID: merged.SourceID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Normalize.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
@@ -377,7 +381,7 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
if err != nil {
return state, fmt.Errorf("encode normalize lane %q: %w", lane.ID, err)
}
diagnostics, err := operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
diagnostics, err := operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
if err != nil {
return state, err
}

View File

@@ -76,6 +76,22 @@ type typedValidateOperation func(context.Context, any, typedValidationTarget) (c
// specializeProducerOperation keeps type erasure inside the registry boundary.
func specializeProducerOperation[T any](operation contracts.ProducerOperation[any]) contracts.ProducerOperation[T] {
result := contracts.ProducerOperation[T]{Controls: operation.Controls, Execution: operation.Execution, ReuseEligible: operation.ReuseEligible}
if cache := operation.Cache; cache != nil {
result.Cache = &contracts.ProducerCache[T]{Store: cache.Store, Observe: cache.Observe}
if cache.Encode != nil {
result.Cache.Encode = func(value T) ([]byte, error) { return cache.Encode(value) }
}
if cache.Decode != nil {
result.Cache.Decode = func(content []byte) (T, error) {
value, err := cache.Decode(content)
if err != nil {
var zero T
return zero, err
}
return exactTypedValue[T]("decode cached artifact", value)
}
}
}
if operation.Assess != nil {
result.Assess = func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
return operation.Assess(ctx, value)

View File

@@ -494,7 +494,7 @@ func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
func registriesFromModuleCatalog(catalog ModuleCatalog) Registries {
return Registries{
Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, ArtifactCodecs: catalog.ArtifactCodecs,
Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, ArtifactCodecs: catalog.ArtifactCodecs, ArtifactEvidence: catalog.ArtifactEvidence,
Extractors: catalog.Extractors, Mergers: catalog.Mergers, Normalizers: catalog.Normalizers,
Validators: catalog.Validators, ValidatorChains: catalog.ValidatorChains, Outputs: catalog.Outputs,
}