379 lines
18 KiB
Go
379 lines
18 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type runnerProbe struct {
|
|
mu sync.Mutex
|
|
calls map[string]int
|
|
eligible map[string][]bool
|
|
}
|
|
|
|
func newRunnerProbe() *runnerProbe {
|
|
return &runnerProbe{calls: map[string]int{}, eligible: map[string][]bool{}}
|
|
}
|
|
func (p *runnerProbe) record(key string, eligible bool) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.calls[key]++
|
|
p.eligible[key] = append(p.eligible[key], eligible)
|
|
}
|
|
|
|
type runnerNotesProducer struct {
|
|
name string
|
|
probe *runnerProbe
|
|
engine *promptkit.Engine
|
|
native bool
|
|
references bool
|
|
failure error
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
func (p *runnerNotesProducer) Key() string { return p.name }
|
|
func (p *runnerNotesProducer) ReferenceSlots() []contracts.ReferenceSlot {
|
|
if !p.references {
|
|
return nil
|
|
}
|
|
return []contracts.ReferenceSlot{{Name: "context", AcceptedArtifactKinds: []contracts.ArtifactKind{"test/notes"}, AcceptedMediaTypes: []string{"application/json"}}}
|
|
}
|
|
func (p *runnerNotesProducer) fallback(ctx context.Context, role string) (codecNotes, error) {
|
|
p.probe.record(p.name+"/"+role+"/fallback", false)
|
|
return codecNotes{Items: []string{}}, ctx.Err()
|
|
}
|
|
func (p *runnerNotesProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) {
|
|
p.probe.record(p.name+"/extract", request.Operation.ReuseEligible)
|
|
if p.cancel != nil {
|
|
p.cancel()
|
|
}
|
|
if p.failure != nil {
|
|
return contracts.TypedExtractionResult[codecNotes]{}, p.failure
|
|
}
|
|
if p.native {
|
|
initial := nativeProducerInitial()
|
|
initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID
|
|
value, outcome, err := RunProducer(ctx, p.engine, request.Operation, initial,
|
|
func(ctx context.Context, result *promptkit.RunResult) (*codecNotes, []string, error) {
|
|
number, _, err := interpretProducerNumber(ctx, result)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return &codecNotes{Items: []string{fmt.Sprint(*number)}}, nil, nil
|
|
}, producerCorrection, func(ctx context.Context, _ *codecNotes) (codecNotes, error) { return p.fallback(ctx, "extract") })
|
|
return contracts.TypedExtractionResult[codecNotes]{Value: value, Outcome: outcome}, err
|
|
}
|
|
return contracts.TypedExtractionResult[codecNotes]{Value: codecNotes{Items: []string{p.name}}, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "extract") }}, nil
|
|
}
|
|
func (p *runnerNotesProducer) Merge(ctx context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
|
p.probe.record(p.name+"/merge", request.Operation.ReuseEligible)
|
|
if p.references {
|
|
if slot, ok := request.References.Slots["context"]; ok && len(slot.Items) > 0 {
|
|
p.probe.record(p.name+"/reference", request.Operation.ReuseEligible)
|
|
}
|
|
}
|
|
value := codecNotes{Items: []string{}}
|
|
for _, output := range request.ExtractOutputs {
|
|
value.Items = append(value.Items, output.Value.Items...)
|
|
}
|
|
return contracts.TypedMergeResult[codecNotes]{Value: value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "merge") }}, nil
|
|
}
|
|
func (p *runnerNotesProducer) Normalize(ctx context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
|
p.probe.record(p.name+"/normalize", request.Operation.ReuseEligible)
|
|
return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "normalize") }}, nil
|
|
}
|
|
|
|
func registerRunnerNotes(t *testing.T, catalog ModuleCatalog, producer runnerNotesProducer) ArtifactLaneProfile {
|
|
t.Helper()
|
|
class := contracts.ExecutionClassDeterministic
|
|
if producer.native {
|
|
class = contracts.ExecutionClassLLMBacked
|
|
}
|
|
spec := func(stage ModuleStage) ModuleSpec {
|
|
return ModuleSpec{Key: producer.name, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes", ReferenceSlots: producer.ReferenceSlots()}
|
|
}
|
|
extractSpec := spec(StageExtract)
|
|
extractSpec.ExecutionClass = class
|
|
if err := RegisterExtractorBuilder(catalog.Extractors, extractSpec, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
|
|
copy := producer
|
|
copy.engine = request.Dependencies.Engine
|
|
return ©, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := RegisterMerger(catalog.Mergers, spec(StageMerge), func() (contracts.Merger[codecNotes], error) { copy := producer; return ©, nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := RegisterNormalizer(catalog.Normalizers, spec(StageNormalize), func() (contracts.Normalizer[codecNotes], error) { copy := producer; return ©, nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return ArtifactLaneProfile{Extract: Binding(producer.name), Merge: Binding(producer.name), Normalize: Binding(producer.name)}
|
|
}
|
|
|
|
func prepareRunnerProfile(t *testing.T, catalog ModuleCatalog, profile PipelineProfile, engine *promptkit.Engine) *PreparedPipeline {
|
|
t.Helper()
|
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resolved, _, err = MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{Engine: engine})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return prepared
|
|
}
|
|
|
|
type namedRunnerValidator struct {
|
|
name string
|
|
notesAssessmentValidator
|
|
}
|
|
|
|
func (v namedRunnerValidator) Name() string { return v.name }
|
|
|
|
func registerRunnerRejection(t *testing.T, catalog ModuleCatalog, probe *runnerProbe) ValidatorOverride {
|
|
t.Helper()
|
|
if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: "reject", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
|
|
return namedRunnerValidator{"reject", notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
probe.record(request.ModuleKey+"/"+request.Stage+"/validator", false)
|
|
return contracts.ValidationResult{ReasonCode: "domain_rejection", CorrectionGuidance: "Return a supported value."}, nil
|
|
})}, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("reject")}}
|
|
}
|
|
|
|
func TestRunnerPropagatesLaneFallbackWithoutCountingDependents(t *testing.T) {
|
|
for _, stage := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
|
t.Run(string(stage), func(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
affected := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "affected", probe: probe})
|
|
unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "unrelated", probe: probe})
|
|
reject := registerRunnerRejection(t, catalog, probe)
|
|
permissive := contracts.ModePermissive
|
|
switch stage {
|
|
case StageExtract:
|
|
affected.Extract.Validators, affected.Extract.Mode = reject, &permissive
|
|
case StageMerge:
|
|
affected.Merge.Validators, affected.Merge.Mode = reject, &permissive
|
|
case StageNormalize:
|
|
affected.Normalize.Validators, affected.Normalize.Mode = reject, &permissive
|
|
}
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"affected": affected, "unrelated": unrelated}
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 2 {
|
|
t.Fatalf("fallback count=%d, outputs=%d", output.FallbackCount, len(output.NormalizeOutputs))
|
|
}
|
|
after := false
|
|
for _, role := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
|
for _, name := range []string{"affected", "unrelated"} {
|
|
key := name + "/" + string(role)
|
|
want := name != "affected" || !after
|
|
if probe.calls[key] != 1 || !reflect.DeepEqual(probe.eligible[key], []bool{want}) {
|
|
t.Fatalf("%s calls=%d eligibility=%#v, want %t", key, probe.calls[key], probe.eligible[key], want)
|
|
}
|
|
}
|
|
if role == stage {
|
|
after = true
|
|
}
|
|
}
|
|
if probe.calls["affected/"+string(stage)+"/validator"] != 1 || probe.calls["affected/"+string(stage)+"/fallback"] != 1 {
|
|
t.Fatalf("deterministic work was retried or fallback revalidated: %#v", probe.calls)
|
|
}
|
|
lanes := output.normalizeReuseEligibility
|
|
for key, eligible := range lanes {
|
|
if eligible != (key.laneID == "unrelated") {
|
|
t.Fatalf("normalized eligibility = %#v", lanes)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerAssessesNativeRevisionsWithoutOuterRevalidation(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
providerCalls := 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
if request.Target.Model != "runner-model" || request.Prompt.SessionID != "runner-session" {
|
|
t.Fatalf("run-wide override missing: %#v", request)
|
|
}
|
|
value := providerCalls
|
|
providerCalls++
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
|
}))
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: probe, native: true})
|
|
for _, name := range []string{"first", "second"} {
|
|
if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
|
|
return namedRunnerValidator{name, notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
|
probe.record(name+"/"+request.Value.Items[0], true)
|
|
if name == "second" && request.Value.Items[0] == "0" {
|
|
return contracts.ValidationResult{ReasonCode: "not_one", CorrectionGuidance: "Return value one."}, nil
|
|
}
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
})}, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first"), Binding("second")}}
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"native": lane}
|
|
profile.LLMProfile = "chosen"
|
|
turns := 1
|
|
profile.MaxAdditionalTurns = &turns
|
|
output, err := New().Run(context.Background(), RunInput{
|
|
Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), SessionID: "runner-session",
|
|
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: "runner-model"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if providerCalls != 2 || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 1 {
|
|
t.Fatalf("native calls=%d, output=%#v", providerCalls, output)
|
|
}
|
|
for _, key := range []string{"first/0", "second/0", "first/1", "second/1", "native/extract", "native/merge", "native/normalize"} {
|
|
if probe.calls[key] != 1 {
|
|
t.Fatalf("%s called %d times", key, probe.calls[key])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "producer", probe: probe})
|
|
earlier := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "earlier", probe: probe})
|
|
consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe, references: true})
|
|
unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "later", probe: probe})
|
|
producer.Normalize.Validators = registerRunnerRejection(t, catalog, probe)
|
|
permissive := contracts.ModePermissive
|
|
producer.Normalize.Mode = &permissive
|
|
consumer.Merge.References = map[string]ReferenceSource{"context": GeneratedReference("build", "producer")}
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = nil
|
|
profile.Steps = []PipelineStepProfile{
|
|
{ID: "build", Artifacts: map[string]ArtifactLaneProfile{"producer": producer, "earlier": earlier}},
|
|
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer, "later": unrelated}},
|
|
}
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 4 || probe.calls["consumer/reference"] != 1 {
|
|
t.Fatalf("handoff failed: count=%d outputs=%d observations=%#v", output.FallbackCount, len(output.NormalizeOutputs), probe.calls)
|
|
}
|
|
for _, key := range []string{"earlier/extract", "earlier/merge", "earlier/normalize", "later/extract", "later/merge", "later/normalize", "consumer/extract"} {
|
|
if !reflect.DeepEqual(probe.eligible[key], []bool{true}) {
|
|
t.Fatalf("unaffected work %s tainted: %#v", key, probe.eligible[key])
|
|
}
|
|
}
|
|
for _, key := range []string{"consumer/merge", "consumer/normalize"} {
|
|
if !reflect.DeepEqual(probe.eligible[key], []bool{false}) {
|
|
t.Fatalf("dependent %s remained reusable: %#v", key, probe.eligible[key])
|
|
}
|
|
}
|
|
if output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "consumer")] || !output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "later")] {
|
|
t.Fatalf("generated lineage = %#v", output.normalizeReuseEligibility)
|
|
}
|
|
}
|
|
|
|
type runnerFallbackChunker struct{}
|
|
|
|
func (runnerFallbackChunker) Key() string { return "fallback-chunk" }
|
|
func (runnerFallbackChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
func (runnerFallbackChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
|
return contracts.ChunkPlanResult{Plan: typedTestPlan(request.Source), Outcome: contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: "selected test fallback", ReuseEligible: false}}, nil
|
|
}
|
|
|
|
func TestRunnerChunkFallbackTaintsEveryDerivedOperation(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})
|
|
profile := typedResolutionProfile()
|
|
profile.Chunk = Binding("fallback-chunk")
|
|
permissive := contracts.ModePermissive
|
|
profile.Chunk.Mode = &permissive
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 1 {
|
|
t.Fatalf("chunk fallback = %#v", output)
|
|
}
|
|
for _, role := range []string{"extract", "merge", "normalize"} {
|
|
if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{false}) {
|
|
t.Fatalf("chunk fallback did not taint %s: %#v", role, probe.eligible)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerDoesNotRetryOrFallbackDeterministicOperationErrors(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
failure := errors.New("deterministic operation failed")
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, failure: failure})
|
|
permissive := contracts.ModePermissive
|
|
profile := typedResolutionProfile()
|
|
profile.Mode = &permissive
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
|
|
_, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
|
|
if !errors.Is(err, failure) || probe.calls["notes/extract"] != 1 || probe.calls["notes/extract/fallback"] != 0 || probe.calls["notes/merge"] != 0 {
|
|
t.Fatalf("operation failure = %v, calls %#v", err, probe.calls)
|
|
}
|
|
}
|
|
|
|
func TestRunnerDoesNotAcceptModuleFallbackUnderStrictControls(t *testing.T) {
|
|
probe := newRunnerProbe()
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile := typedResolutionProfile()
|
|
profile.Chunk = Binding("fallback-chunk")
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})}
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
|
|
if err == nil || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 || probe.calls["notes/extract"] != 0 {
|
|
t.Fatalf("strict controls accepted module fallback: %#v %v", output, err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerDiscardsDeterministicOutputReturnedAfterCancellation(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
probe := newRunnerProbe()
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, cancel: cancel})
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
|
|
output, err := New().Run(ctx, RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
|
|
if !errors.Is(err, context.Canceled) || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 || probe.calls["notes/merge"] != 0 || probe.calls["notes/extract/fallback"] != 0 {
|
|
t.Fatalf("cancellation = %#v %v, calls %#v", output, err, probe.calls)
|
|
}
|
|
}
|
|
|
|
var _ contracts.Chunker = runnerFallbackChunker{}
|
|
var _ contracts.Extractor[codecNotes] = (*runnerNotesProducer)(nil)
|
|
var _ contracts.Merger[codecNotes] = (*runnerNotesProducer)(nil)
|
|
var _ contracts.Normalizer[codecNotes] = (*runnerNotesProducer)(nil)
|