Files
notarius/internal/framework/pipeline/evidence_output_test.go

254 lines
11 KiB
Go

package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"go.maximumdirect.net/promptkit"
)
type capturingEvidenceOutput struct {
requests []contracts.OutputRequest
}
func (*capturingEvidenceOutput) Key() string { return "capture/evidence-context" }
func (output *capturingEvidenceOutput) Encode(_ context.Context, request contracts.OutputRequest) (contracts.OutputResult, error) {
request.EvidenceContext = contracts.CloneSerializedArtifactPointer(request.EvidenceContext)
output.requests = append(output.requests, request)
return contracts.OutputResult{}, nil
}
func installEvidencePlan(prepared *PreparedPipeline, window int, selected []string, project func(codecNotes) ([]source.SourceRef, error)) {
lanes := make([]preparedEvidenceLane, 0, len(selected))
for _, laneID := range selected {
for _, step := range prepared.Steps {
for _, lane := range step.lanes {
if lane.resolved.ID != laneID {
continue
}
lanes = append(lanes, preparedEvidenceLane{
laneID: laneID,
kind: lane.resolved.ArtifactKind,
project: func(value any) ([]source.SourceRef, error) {
notes, ok := value.(codecNotes)
if !ok {
return nil, errors.New("unexpected artifact type")
}
return project(notes)
},
})
}
}
}
prepared.evidencePlan = &preparedEvidencePlan{policy: EvidenceContextPolicy{Enabled: true, WindowUnits: window, LaneIDs: append([]string(nil), selected...)}, lanes: lanes}
}
func setNormalizedNotes(prepared *PreparedPipeline, values map[string]codecNotes) {
for stepIndex := range prepared.Steps {
for laneIndex := range prepared.Steps[stepIndex].lanes {
lane := &prepared.Steps[stepIndex].lanes[laneIndex]
value, ok := values[lane.resolved.ID]
if !ok {
continue
}
lane.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
}
}
}
func decodeCapturedEvidence(t *testing.T, output *capturingEvidenceOutput) evidencecontext.Document {
t.Helper()
if len(output.requests) != 1 || output.requests[0].EvidenceContext == nil {
t.Fatalf("output requests = %#v, want one evidence context", output.requests)
}
value, err := evidencecontext.New().Decode(output.requests[0].EvidenceContext.Content)
if err != nil {
t.Fatalf("Decode(evidence context): %v", err)
}
return value
}
func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs(t *testing.T) {
prepared := preparedOrderedPipeline(t, 3, orderedLaneSpec{id: "alpha", profile: "notes"}, orderedLaneSpec{id: "beta", profile: "notes"})
encoder := &capturingEvidenceOutput{}
prepared.output = encoder
setNormalizedNotes(prepared, map[string]codecNotes{"alpha": {Items: []string{"one"}}, "beta": {Items: []string{"two"}}})
installEvidencePlan(prepared, 1, []string{"alpha", "beta", "inactive"}, func(notes codecNotes) ([]source.SourceRef, error) {
switch notes.Items[0] {
case "one":
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil
case "two":
return []source.SourceRef{{SourceID: "source", StartUnitID: 2, EndUnitID: 2}}, nil
default:
return nil, nil
}
})
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v", err)
}
value := decodeCapturedEvidence(t, encoder)
if actual := []int{value[0].ID, value[1].ID, value[2].ID}; !reflect.DeepEqual(actual, []int{1, 2, 3}) {
t.Fatalf("evidence context = %#v, want selected union", value)
}
}
func TestEvidenceContextOmitsUnselectedAndAbsentLanes(t *testing.T) {
prepared := preparedOrderedPipeline(t, 2, orderedLaneSpec{id: "present", profile: "notes"}, orderedLaneSpec{id: "unselected", profile: "notes"})
doc := prepared.input.(*typedTestInput).doc
installEvidencePlan(prepared, 0, []string{"absent", "present"}, func(codecNotes) ([]source.SourceRef, error) {
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}, nil
})
artifact, err := prepared.artifactCodecs.Encode("test/notes", codecNotes{Items: []string{"accepted"}})
if err != nil {
t.Fatal(err)
}
result, _, err := buildOutputEvidenceContext(prepared, doc, []contracts.SerializedOutput{{LaneID: "present", SourceID: doc.ID, Artifact: artifact}, {LaneID: "unselected", SourceID: doc.ID, Artifact: artifact}})
if err != nil {
t.Fatal(err)
}
value, err := evidencecontext.New().Decode(result.Content)
if err != nil || len(value) != 1 || value[0].ID != 1 {
t.Fatalf("evidence=%#v %v", value, err)
}
}
func TestRunnerEvidenceContextFailurePreventsOutputEncoding(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
encoder := &capturingEvidenceOutput{}
prepared.output = encoder
setNormalizedNotes(prepared, map[string]codecNotes{"notes": {Items: []string{"invalid"}}})
installEvidencePlan(prepared, 0, []string{"notes"}, func(codecNotes) ([]source.SourceRef, error) {
return []source.SourceRef{{SourceID: "source", StartUnitID: 99, EndUnitID: 99}}, nil
})
result, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err == nil || !strings.Contains(err.Error(), "accepted evidence references are invalid") {
t.Fatalf("Run() error = %v, want evidence context failure", err)
}
if len(encoder.requests) != 0 || result.ArtifactCount() != 0 {
t.Fatalf("invalid evidence was published: %#v", result)
}
}
func TestBuildOutputEvidenceContextRejectsIncompatibleAcceptedOutputs(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
doc := prepared.input.(*typedTestInput).doc
installEvidencePlan(prepared, 0, []string{"notes"}, func(codecNotes) ([]source.SourceRef, error) {
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}, nil
})
artifact, err := prepared.artifactCodecs.Encode("test/notes", codecNotes{Items: []string{"valid"}})
if err != nil {
t.Fatal(err)
}
valid := contracts.SerializedOutput{LaneID: "notes", SourceID: doc.ID, Artifact: artifact}
for _, test := range []struct {
name string
outputs []contracts.SerializedOutput
mutate func(*contracts.SerializedOutput)
want string
}{
{name: "duplicate lane", outputs: []contracts.SerializedOutput{valid, valid}, want: "duplicate lane"},
{name: "foreign source", outputs: []contracts.SerializedOutput{valid}, mutate: func(output *contracts.SerializedOutput) { output.SourceID = "other" }, want: "source is incompatible"},
{name: "wrong artifact kind", outputs: []contracts.SerializedOutput{valid}, mutate: func(output *contracts.SerializedOutput) { output.Artifact.Kind = "test/score" }, want: "artifact kind is incompatible"},
{name: "invalid artifact payload", outputs: []contracts.SerializedOutput{valid}, mutate: func(output *contracts.SerializedOutput) { output.Artifact.Content = []byte("not JSON") }, want: "cannot be decoded"},
} {
t.Run(test.name, func(t *testing.T) {
outputs := append([]contracts.SerializedOutput(nil), test.outputs...)
for index := range outputs {
outputs[index] = contracts.CloneSerializedOutput(outputs[index])
}
if test.mutate != nil {
test.mutate(&outputs[0])
}
_, _, err := buildOutputEvidenceContext(prepared, doc, outputs)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("buildOutputEvidenceContext() error = %v, want %q", err, test.want)
}
if strings.Contains(err.Error(), "not JSON") {
t.Fatalf("buildOutputEvidenceContext() exposed artifact payload: %v", err)
}
})
}
}
func TestEvidenceContextOutputRequestOwnsArtifactBytes(t *testing.T) {
artifact := &contracts.SerializedArtifact{Content: []byte("original"), Metadata: map[string]any{"source": "original"}}
request := contracts.OutputRequest{EvidenceContext: contracts.CloneSerializedArtifactPointer(artifact)}
request.EvidenceContext.Content[0] = 'X'
request.EvidenceContext.Metadata["source"] = "changed"
if string(artifact.Content) != "original" || artifact.Metadata["source"] != "original" {
t.Fatalf("output request evidence context aliases source artifact: %#v", artifact)
}
}
func TestRunnerEvidenceContextRebuildsFromCachedNormalizedArtifact(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
encoder := &capturingEvidenceOutput{}
prepared.output = encoder
doc := prepared.input.(*typedTestInput).doc
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
lane := &prepared.Steps[0].lanes[0]
lane.resolved.NormalizeExecutionClass = contracts.ExecutionClassLLMBacked
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
operation := request.Operation
assess := operation.Assess
operation.Assess = func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
assessments++
return assess(ctx, value)
}
value, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*any, []string, error) {
var value any = codecNotes{Items: []string{"stored"}}
return &value, nil, nil
}, nil, nil)
return erasedTypedResult{Value: value, Outcome: outcome}, err
}
installEvidencePlan(prepared, 0, []string{"notes"}, func(notes codecNotes) ([]source.SourceRef, error) {
if !reflect.DeepEqual(notes.Items, []string{"stored"}) {
return nil, errors.New("cached artifact was not projected")
}
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}, nil
})
_, store := filesystemProducerCache(t)
for range 2 {
encoder.requests = nil
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Workers: 1}); err != nil {
t.Fatal(err)
}
value := decodeCapturedEvidence(t, encoder)
if len(value) != 1 || value[0].ID != 1 {
t.Fatalf("evidence=%#v", value)
}
}
if calls != 1 || assessments != 1 {
t.Fatalf("cache reuse calls=%d assessments=%d", calls, assessments)
}
}
func TestRunnerSkipsEvidenceContextWhenOutputDoesNotOptIn(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
encoder := &capturingEvidenceOutput{}
prepared.output = encoder
projected := 0
prepared.evidencePlan = nil
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatalf("Run() error = %v", err)
}
if projected != 0 || len(encoder.requests) != 1 || encoder.requests[0].EvidenceContext != nil {
t.Fatalf("projected = %d requests = %#v, want no evidence work", projected, encoder.requests)
}
}