Capture native execution facts and complete scoped debug evidence
This commit is contained in:
@@ -37,6 +37,7 @@ type CacheObservation struct {
|
||||
Decision string
|
||||
MessageContentHash string
|
||||
Prepared *promptkit.PreparedRun
|
||||
Artifact []byte `json:"-"`
|
||||
Err error
|
||||
ChunkID string
|
||||
ChunkIndex *int
|
||||
@@ -65,8 +66,35 @@ type ExecutionOptions struct {
|
||||
TransientRetries *int
|
||||
Observer promptkit.ExecutionObserver `json:"-"`
|
||||
OnPrepared func(context.Context, promptkit.PreparedRun) `json:"-"`
|
||||
Capture EvidenceWriter `json:"-"`
|
||||
}
|
||||
|
||||
// EvidenceWriter receives optional operation-scoped debug evidence. Modules get
|
||||
// no filesystem root and cannot use capture as an execution or reporting service.
|
||||
type EvidenceWriter interface {
|
||||
WriteJSON(string, any) error
|
||||
WriteBytes(string, []byte) error
|
||||
}
|
||||
|
||||
// RecordEvidence writes available application-owned evidence at an ordinary
|
||||
// operation boundary. A recording failure must not become semantic fallback.
|
||||
func (options ExecutionOptions) RecordEvidence(name string, value any) error {
|
||||
if options.Capture == nil {
|
||||
return nil
|
||||
}
|
||||
if err := options.Capture.WriteJSON(name, value); err != nil {
|
||||
return &CaptureError{Cause: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CaptureError is an ordinary framework failure, not a semantic assessment or a
|
||||
// reason to invoke a model again. Its cause remains available to the caller.
|
||||
type CaptureError struct{ Cause error }
|
||||
|
||||
func (e *CaptureError) Error() string { return "record debug evidence: " + e.Cause.Error() }
|
||||
func (e *CaptureError) Unwrap() error { return e.Cause }
|
||||
|
||||
type AssessmentDecision string
|
||||
|
||||
const (
|
||||
|
||||
@@ -6,7 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
@@ -30,6 +33,10 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i
|
||||
}
|
||||
validator.position = index + 1
|
||||
result, err := invoke(ctx, validator)
|
||||
var captureErr *contracts.CaptureError
|
||||
if errors.As(err, &captureErr) {
|
||||
return assessment, errors.Join(ctx.Err(), err)
|
||||
}
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return assessment, canceled
|
||||
}
|
||||
@@ -70,6 +77,7 @@ func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
|
||||
// typedCandidateAssessor binds one operation's complete chain. Each invocation
|
||||
// captures the new candidate once and decodes a detached view for each validator.
|
||||
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
|
||||
var candidateNumber atomic.Uint64
|
||||
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
||||
if ctx == nil {
|
||||
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||
@@ -81,12 +89,21 @@ func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidat
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
|
||||
}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
|
||||
if capture != nil {
|
||||
if err := capture.WriteBytes("candidate.json", content); err != nil {
|
||||
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
|
||||
}
|
||||
}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
|
||||
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
|
||||
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
|
||||
request, err := isolatedValidationTarget(target)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||
}
|
||||
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||
request.execution.Capture = validatorCapture
|
||||
switch validator.resolved.Target {
|
||||
case ValidatorTargetTyped:
|
||||
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
|
||||
@@ -153,6 +170,7 @@ func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument,
|
||||
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
|
||||
// The runner's plan callback owns materialization before this boundary.
|
||||
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
|
||||
var candidateNumber atomic.Uint64
|
||||
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
|
||||
if ctx == nil {
|
||||
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
|
||||
@@ -167,12 +185,21 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
|
||||
target := target
|
||||
target.chunks = chunks
|
||||
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
|
||||
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
|
||||
if capture != nil {
|
||||
if err := capture.WriteBytes("candidate.json", content); err != nil {
|
||||
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
|
||||
}
|
||||
}
|
||||
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
|
||||
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
|
||||
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
|
||||
request, err := isolatedValidationTarget(target)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
|
||||
}
|
||||
request.llmProfile = validator.resolved.Binding.LLMProfile
|
||||
request.execution.Capture = validatorCapture
|
||||
switch validator.resolved.Target {
|
||||
case ValidatorTargetChunk:
|
||||
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
|
||||
@@ -193,3 +220,17 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func captureValidationFinding(writer contracts.EvidenceWriter, finding contracts.ValidationResult, failure error) error {
|
||||
if writer == nil {
|
||||
return failure
|
||||
}
|
||||
errorText := ""
|
||||
if failure != nil {
|
||||
errorText = failure.Error()
|
||||
}
|
||||
if err := writer.WriteJSON("finding.json", map[string]any{"result": finding, "correction_guidance": finding.CorrectionGuidance, "error": errorText}); err != nil {
|
||||
return errors.Join(failure, &contracts.CaptureError{Cause: err})
|
||||
}
|
||||
return failure
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
@@ -124,6 +125,56 @@ func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
|
||||
return entry
|
||||
}
|
||||
|
||||
func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
content := strings.Repeat("sk-ordinary content ", 1000)
|
||||
candidate := codecNotes{Items: []string{content}}
|
||||
validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
|
||||
if request.Execution.Capture == nil {
|
||||
t.Fatal("validator has no scoped capture")
|
||||
}
|
||||
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil
|
||||
})
|
||||
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}})
|
||||
for range 2 {
|
||||
assessment, err := assess(context.Background(), candidate)
|
||||
if err != nil || assessment.Accepted() {
|
||||
t.Fatalf("assessment=%#v %v", assessment, err)
|
||||
}
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 {
|
||||
t.Fatal("revised candidates or findings were overwritten")
|
||||
}
|
||||
for name, data := range evidence.files {
|
||||
if strings.HasSuffix(name, "/candidate.json") {
|
||||
decoded, err := assessmentNotesCodec(t).decodeCandidate(data)
|
||||
if err != nil || !reflect.DeepEqual(decoded, candidate) {
|
||||
t.Fatalf("codec candidate changed: %#v, %v", decoded, err)
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(name, "/finding.json") {
|
||||
var finding struct {
|
||||
Guidance string `json:"correction_guidance"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content {
|
||||
t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) {
|
||||
primary := errors.New("capture failed")
|
||||
called := 0
|
||||
_, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
|
||||
called++
|
||||
return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary}
|
||||
})
|
||||
if called != 1 || !errors.Is(err, primary) {
|
||||
t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
|
||||
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
|
||||
doc := typedTestDocument()
|
||||
|
||||
@@ -1,22 +1,13 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
@@ -36,21 +27,19 @@ func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
|
||||
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
|
||||
|
||||
type debugTimedEnvelope struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
AttemptKind string `json:"attempt_kind,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Payload any `json:"payload,omitempty"`
|
||||
LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Payload any `json:"payload,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugBinaryEnvelope struct {
|
||||
Text string `json:"text,omitempty"`
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
@@ -61,7 +50,6 @@ type debugSourceInput struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw debugBinaryEnvelope `json:"raw,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
@@ -109,244 +97,6 @@ type debugSerializedOutput struct {
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
|
||||
|
||||
type debugStructuredCompletionResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMPromptArtifact struct {
|
||||
CallID string `json:"call_id"`
|
||||
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMResponseArtifact struct {
|
||||
CallID string `json:"call_id"`
|
||||
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
|
||||
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
|
||||
ContentPath string `json:"content_path,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMCallReference struct {
|
||||
CallID string `json:"call_id"`
|
||||
PromptPath string `json:"prompt_path,omitempty"`
|
||||
ResponsePath string `json:"response_path"`
|
||||
ResponseContentPath string `json:"response_content_path,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||
Error bool `json:"error,omitempty"`
|
||||
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationCall struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Request any `json:"request"`
|
||||
Result contracts.ValidationResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMClient struct {
|
||||
inner contracts.StructuredLLMClient
|
||||
recorder DebugRecorder
|
||||
mu sync.Mutex
|
||||
counter int
|
||||
}
|
||||
|
||||
type debugLLMScope struct {
|
||||
prefix string
|
||||
parent *debugLLMScope
|
||||
mu sync.Mutex
|
||||
calls []debugLLMCallReference
|
||||
}
|
||||
|
||||
type debugLLMScopeContextKey struct{}
|
||||
|
||||
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
|
||||
if client == nil || recorder == nil || !recorder.Enabled() {
|
||||
return client
|
||||
}
|
||||
if _, ok := client.(*debugLLMClient); ok {
|
||||
return client
|
||||
}
|
||||
return &debugLLMClient{inner: client, recorder: synchronizedDebugRecorder(recorder)}
|
||||
}
|
||||
|
||||
// WithDebugLLMRecording decorates a shared LLM client so calls made by
|
||||
// construction-injected modules participate in the run's debug recording.
|
||||
func WithDebugLLMRecording(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
|
||||
return wrapDebugLLMClient(client, recorder)
|
||||
}
|
||||
|
||||
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.mu.Lock()
|
||||
client.counter++
|
||||
callID := fmt.Sprintf("%04d", client.counter)
|
||||
client.mu.Unlock()
|
||||
|
||||
started := time.Now().UTC()
|
||||
response, err := client.inner.CompleteStructured(ctx, req, out)
|
||||
completed := time.Now().UTC()
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
|
||||
scopePrefix := cleanDebugPath(req.StageName)
|
||||
if req.StageName == "" {
|
||||
scopePrefix = "llm"
|
||||
}
|
||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||
scopePrefix = scope.prefix
|
||||
}
|
||||
promptPath := ""
|
||||
var writeErr error
|
||||
if response.Debug != nil && response.Debug.Prompt != nil {
|
||||
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
|
||||
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
|
||||
Stage: req.StageName,
|
||||
ModuleKey: req.StageName,
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
DurationMS: completed.Sub(started).Milliseconds(),
|
||||
Payload: debugLLMPromptArtifact{
|
||||
CallID: callID,
|
||||
Prompt: response.Debug.Prompt,
|
||||
},
|
||||
}))
|
||||
}
|
||||
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
|
||||
responseMaterial := debugResponseMaterial(response)
|
||||
fallbackMaterial := debugCompletionFallback(response)
|
||||
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
|
||||
writeErr = errors.Join(writeErr, contentErr)
|
||||
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
|
||||
Stage: req.StageName,
|
||||
ModuleKey: req.StageName,
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
DurationMS: completed.Sub(started).Milliseconds(),
|
||||
Payload: debugLLMResponseArtifact{
|
||||
CallID: callID,
|
||||
Response: responseForArtifact,
|
||||
Fallback: fallbackForArtifact,
|
||||
ContentPath: responseContentPath,
|
||||
Error: errorText,
|
||||
Execution: cloneLLMExecutionSummary(response.Execution),
|
||||
},
|
||||
Error: errorText,
|
||||
}))
|
||||
callRef := debugLLMCallReference{
|
||||
CallID: callID,
|
||||
PromptPath: promptPath,
|
||||
ResponsePath: responsePath,
|
||||
ResponseContentPath: responseContentPath,
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
|
||||
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
|
||||
PromptTokens: response.PromptTokens,
|
||||
CompletionTokens: response.CompletionTokens,
|
||||
TotalTokens: response.TotalTokens,
|
||||
RepairAttempts: response.RepairAttempts,
|
||||
Error: err != nil,
|
||||
Execution: cloneLLMExecutionSummary(response.Execution),
|
||||
}
|
||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||
scope.record(callRef)
|
||||
}
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
if writeErr != nil {
|
||||
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
|
||||
}
|
||||
return response, err
|
||||
}
|
||||
|
||||
func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
scope := &debugLLMScope{
|
||||
prefix: prefix,
|
||||
parent: debugLLMScopeFromContext(ctx),
|
||||
}
|
||||
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
|
||||
}
|
||||
|
||||
func withIsolatedDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
scope := &debugLLMScope{prefix: prefix}
|
||||
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
|
||||
}
|
||||
|
||||
func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope)
|
||||
return scope
|
||||
}
|
||||
|
||||
func (scope *debugLLMScope) record(ref debugLLMCallReference) {
|
||||
if scope == nil {
|
||||
return
|
||||
}
|
||||
scope.mu.Lock()
|
||||
scope.calls = append(scope.calls, ref)
|
||||
scope.mu.Unlock()
|
||||
if scope.parent != nil {
|
||||
scope.parent.record(ref)
|
||||
}
|
||||
}
|
||||
|
||||
func (scope *debugLLMScope) references() []debugLLMCallReference {
|
||||
if scope == nil {
|
||||
return nil
|
||||
}
|
||||
scope.mu.Lock()
|
||||
defer scope.mu.Unlock()
|
||||
if len(scope.calls) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugLLMCallReference, len(scope.calls))
|
||||
copy(out, scope.calls)
|
||||
return out
|
||||
}
|
||||
|
||||
func cleanDebugPath(value string) string {
|
||||
parts := strings.Split(value, "/")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
out = append(out, fileio.EncodePathComponent(part))
|
||||
}
|
||||
return strings.Join(out, "/")
|
||||
}
|
||||
|
||||
func debugFirstNonEmptyString(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
|
||||
if recorder == nil || !recorder.Enabled() {
|
||||
return nil
|
||||
@@ -363,60 +113,17 @@ func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnv
|
||||
return recorder.WriteJSON(name, envelope)
|
||||
}
|
||||
|
||||
func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope {
|
||||
if scope != nil {
|
||||
envelope.LLMCalls = scope.references()
|
||||
}
|
||||
return envelope
|
||||
}
|
||||
|
||||
func writeDebugAttempt(recorder DebugRecorder, attemptPath string, envelope debugTimedEnvelope, scope *debugLLMScope) error {
|
||||
return writeDebugTimed(recorder, attemptPath+".json", debugEnvelopeWithLLMCalls(envelope, scope))
|
||||
}
|
||||
|
||||
type attemptTerminalRecorder struct {
|
||||
recorder DebugRecorder
|
||||
path string
|
||||
label string
|
||||
scope *debugLLMScope
|
||||
envelope debugTimedEnvelope
|
||||
}
|
||||
|
||||
type attemptDebugPersistenceError struct {
|
||||
label string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *attemptDebugPersistenceError) Error() string {
|
||||
return fmt.Sprintf("write %s attempt debug artifact: %v", e.label, e.err)
|
||||
}
|
||||
|
||||
func (e *attemptDebugPersistenceError) Unwrap() error { return e.err }
|
||||
|
||||
func newAttemptTerminalRecorder(recorder DebugRecorder, attemptPath, label string, scope *debugLLMScope, envelope debugTimedEnvelope) attemptTerminalRecorder {
|
||||
return attemptTerminalRecorder{recorder: recorder, path: attemptPath, label: label, scope: scope, envelope: envelope}
|
||||
}
|
||||
|
||||
func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
|
||||
envelope := r.envelope
|
||||
envelope.Payload = payload
|
||||
if terminalErr != nil {
|
||||
envelope.Error = terminalErr.Error()
|
||||
}
|
||||
if err := writeDebugAttempt(r.recorder, r.path, envelope, r.scope); err != nil {
|
||||
debugErr := &attemptDebugPersistenceError{label: r.label, err: err}
|
||||
return errors.Join(terminalErr, debugErr)
|
||||
}
|
||||
return terminalErr
|
||||
}
|
||||
|
||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
|
||||
content = redactSecretBytes(content)
|
||||
text := ""
|
||||
if utf8.Valid(content) {
|
||||
text = string(content)
|
||||
}
|
||||
return debugBinaryEnvelope{
|
||||
Text: text,
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: debugContentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: redactSensitiveMap(metadata),
|
||||
Metadata: metadata,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +137,7 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
|
||||
Format: doc.Format,
|
||||
Digest: doc.Digest,
|
||||
Units: cloneSourceUnitsForDebug(doc.Units),
|
||||
Metadata: redactSensitiveMap(doc.Metadata),
|
||||
Metadata: doc.Metadata,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,7 +149,7 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
|
||||
Ref: chunk.Ref,
|
||||
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnitsForDebug(chunk.Units),
|
||||
Metadata: redactSensitiveMap(chunk.Metadata),
|
||||
Metadata: chunk.Metadata,
|
||||
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
|
||||
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
|
||||
}
|
||||
@@ -457,9 +164,9 @@ func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
|
||||
cloned[i] = source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: unit.Kind,
|
||||
Text: string(redactSecretBytes([]byte(unit.Text))),
|
||||
Text: unit.Text,
|
||||
Ref: unit.Ref,
|
||||
Metadata: redactSensitiveMap(unit.Metadata),
|
||||
Metadata: unit.Metadata,
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
@@ -510,7 +217,6 @@ func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
|
||||
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
|
||||
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
||||
digest := contracts.DigestArtifactSchema(schema)
|
||||
schema.JSONSchema = nil
|
||||
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
|
||||
content.ContentDigest = debugContentDigest(output.Artifact.Content)
|
||||
return debugSerializedOutput{
|
||||
@@ -552,189 +258,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
||||
return out
|
||||
}
|
||||
|
||||
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
||||
return req.DebugSummary()
|
||||
}
|
||||
|
||||
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
||||
return debugStructuredCompletionResponse{
|
||||
Content: string(response.Content),
|
||||
Provider: response.Provider,
|
||||
Model: response.Model,
|
||||
ProfileID: response.ProfileID,
|
||||
PromptTokens: response.PromptTokens,
|
||||
CompletionTokens: response.CompletionTokens,
|
||||
TotalTokens: response.TotalTokens,
|
||||
Execution: cloneLLMExecutionSummary(response.Execution),
|
||||
}
|
||||
}
|
||||
|
||||
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
|
||||
if response.Debug != nil && response.Debug.Response != nil {
|
||||
return nil
|
||||
}
|
||||
fallback := debugCompletionResponse(response)
|
||||
if fallback.Content == "" &&
|
||||
fallback.Provider == "" &&
|
||||
fallback.Model == "" &&
|
||||
fallback.ProfileID == "" &&
|
||||
fallback.PromptTokens == 0 &&
|
||||
fallback.CompletionTokens == 0 &&
|
||||
fallback.TotalTokens == 0 &&
|
||||
fallback.Execution == nil {
|
||||
return nil
|
||||
}
|
||||
return &fallback
|
||||
}
|
||||
|
||||
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
|
||||
responseCopy := cloneDebugResponseWithoutContent(response)
|
||||
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
|
||||
content := ""
|
||||
if response != nil {
|
||||
content = response.Content
|
||||
}
|
||||
if content == "" && fallback != nil {
|
||||
content = fallback.Content
|
||||
}
|
||||
if content == "" {
|
||||
return "", responseCopy, fallbackCopy, nil
|
||||
}
|
||||
|
||||
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
|
||||
if recorder == nil || !recorder.Enabled() {
|
||||
return contentPath, responseCopy, fallbackCopy, nil
|
||||
}
|
||||
if err := recorder.WriteBytes(contentPath, data); err != nil {
|
||||
return contentPath, responseCopy, fallbackCopy, err
|
||||
}
|
||||
return contentPath, responseCopy, fallbackCopy, nil
|
||||
}
|
||||
|
||||
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
|
||||
if response == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *response
|
||||
clone.Content = ""
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
|
||||
if fallback == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *fallback
|
||||
clone.Content = ""
|
||||
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
|
||||
if summary == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *summary
|
||||
return &clone
|
||||
}
|
||||
|
||||
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
|
||||
raw := []byte(content)
|
||||
if json.Valid(raw) {
|
||||
var formatted bytes.Buffer
|
||||
if err := json.Indent(&formatted, raw, "", " "); err == nil {
|
||||
formatted.WriteByte('\n')
|
||||
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
|
||||
}
|
||||
}
|
||||
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
|
||||
}
|
||||
|
||||
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
|
||||
if response.Debug == nil {
|
||||
return nil
|
||||
}
|
||||
return response.Debug.Response
|
||||
}
|
||||
|
||||
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
|
||||
if response.Debug == nil || response.Debug.Response == nil {
|
||||
return ""
|
||||
}
|
||||
return response.Debug.Response.ModelName
|
||||
}
|
||||
|
||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||
result.CorrectionGuidance = ""
|
||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||
return result
|
||||
}
|
||||
|
||||
func debugContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
var secretPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
|
||||
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
|
||||
}
|
||||
|
||||
func redactSecretBytes(content []byte) []byte {
|
||||
if len(content) == 0 || !utf8.Valid(content) {
|
||||
return append([]byte(nil), content...)
|
||||
}
|
||||
text := string(content)
|
||||
for _, pattern := range secretPatterns {
|
||||
text = pattern.ReplaceAllString(text, "[REDACTED]")
|
||||
}
|
||||
return []byte(text)
|
||||
}
|
||||
|
||||
func redactSensitiveMap(values map[string]any) map[string]any {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
out[key] = redactSensitiveValue(value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactSensitiveValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return string(redactSecretBytes([]byte(typed)))
|
||||
case map[string]any:
|
||||
return redactSensitiveMap(typed)
|
||||
case map[string]string:
|
||||
out := make(map[string]string, len(typed))
|
||||
for key, value := range typed {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
} else {
|
||||
out[key] = string(redactSecretBytes([]byte(value)))
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
key = strings.ToLower(key)
|
||||
return strings.Contains(key, "api_key") ||
|
||||
strings.Contains(key, "apikey") ||
|
||||
strings.Contains(key, "authorization") ||
|
||||
strings.Contains(key, "bearer") ||
|
||||
strings.Contains(key, "password") ||
|
||||
strings.Contains(key, "secret") ||
|
||||
strings.Contains(key, "token")
|
||||
}
|
||||
|
||||
255
internal/framework/pipeline/native_capture.go
Normal file
255
internal/framework/pipeline/native_capture.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// NativeCapture retains only one request's prepared snapshot. Events are written
|
||||
// incrementally, not accumulated as a second execution/usage ledger. It never
|
||||
// prepares or executes a request itself.
|
||||
type NativeCapture struct {
|
||||
writer contracts.EvidenceWriter
|
||||
mu sync.Mutex
|
||||
prepared *promptkit.PreparedRun
|
||||
runID string
|
||||
events uint64
|
||||
err error
|
||||
}
|
||||
|
||||
// CaptureNativeRequest composes existing hooks and enables native exchange
|
||||
// capture only when evidence was requested. Request credentials remain excluded
|
||||
// by the native request's own JSON representation.
|
||||
func CaptureNativeRequest(request promptkit.RunRequest, writer contracts.EvidenceWriter) (promptkit.RunRequest, *NativeCapture) {
|
||||
if writer == nil {
|
||||
return request, nil
|
||||
}
|
||||
capture := &NativeCapture{writer: writer}
|
||||
capture.remember(writer.WriteJSON("request.json", request))
|
||||
request.CaptureExchange = true
|
||||
existing := request.OnPrepared
|
||||
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
|
||||
// No serialization or filesystem I/O while native admission is held.
|
||||
// Native hands off a detached value. Copy mutable fields only when an
|
||||
// existing callback also owns that value and may change it.
|
||||
if existing != nil {
|
||||
capture.RetainPrepared(copyPreparedEvidence(details))
|
||||
existing(ctx, details)
|
||||
} else {
|
||||
capture.RetainPrepared(details)
|
||||
}
|
||||
}
|
||||
request.Observer = promptkit.ComposeObservers(request.Observer, capture.observe)
|
||||
return request, capture
|
||||
}
|
||||
|
||||
// RetainPrepared uses the already available split-preparation details on cache
|
||||
// paths, including hits on which OnPrepared and native events never occur.
|
||||
func (capture *NativeCapture) RetainPrepared(details promptkit.PreparedRun) {
|
||||
if capture == nil {
|
||||
return
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
capture.prepared = &details
|
||||
}
|
||||
|
||||
func (capture *NativeCapture) remember(err error) {
|
||||
if capture == nil || err == nil {
|
||||
return
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
if capture.err == nil {
|
||||
capture.err = err
|
||||
}
|
||||
}
|
||||
|
||||
func (capture *NativeCapture) observe(event promptkit.ExecutionEvent) {
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
capture.runID = event.RunID
|
||||
capture.events++
|
||||
err := capture.writer.WriteJSON(path.Join("native", event.RunID, "events", fmt.Sprintf("%06d.json", capture.events)), event)
|
||||
if capture.err == nil {
|
||||
capture.err = err
|
||||
}
|
||||
}
|
||||
|
||||
// Err exposes synchronous callback/write failures at the next normal boundary.
|
||||
func (capture *NativeCapture) Err() error {
|
||||
if capture == nil {
|
||||
return nil
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
if capture.err == nil {
|
||||
return nil
|
||||
}
|
||||
return &contracts.CaptureError{Cause: capture.err}
|
||||
}
|
||||
|
||||
// Finish records available evidence even when native execution returned no
|
||||
// result. Unassessed marks a response withheld from the application by native
|
||||
// coordination; cacheHit never fabricates an execution or historical result.
|
||||
func (capture *NativeCapture) Finish(result *promptkit.RunResult, nativeErr error, unassessed, cacheHit bool) error {
|
||||
if capture == nil {
|
||||
return nil
|
||||
}
|
||||
capture.mu.Lock()
|
||||
prepared, runID := capture.prepared, capture.runID
|
||||
capture.prepared = nil
|
||||
capture.mu.Unlock()
|
||||
if result != nil {
|
||||
runID = result.RunID
|
||||
}
|
||||
prefix := "unexecuted"
|
||||
if runID != "" {
|
||||
prefix = path.Join("native", runID)
|
||||
}
|
||||
writeJSON := func(name string, value any) {
|
||||
capture.remember(capture.writer.WriteJSON(path.Join(prefix, name), value))
|
||||
}
|
||||
writeText := func(name, value string) {
|
||||
capture.remember(capture.writer.WriteBytes(path.Join(prefix, name), []byte(value)))
|
||||
}
|
||||
if prepared != nil {
|
||||
writeJSON("prepared.json", prepared)
|
||||
writeText("prepared-messages.txt", readableNativeMessages("Prepared request messages", prepared.Messages))
|
||||
}
|
||||
if result != nil {
|
||||
writeJSON("result.json", result)
|
||||
writeText("response.txt", result.RawOutput)
|
||||
if result.Exchange != nil {
|
||||
details := result.Exchange.Details()
|
||||
writeJSON("exchange.json", details)
|
||||
writeText("messages.txt", readableNativeMessages("Completed exchange", details.Messages))
|
||||
}
|
||||
}
|
||||
errorText := ""
|
||||
if nativeErr != nil {
|
||||
errorText = nativeErr.Error()
|
||||
}
|
||||
writeJSON("availability.json", map[string]any{
|
||||
"run_id": runID, "prepared_available": prepared != nil,
|
||||
"result_available": result != nil, "unassessed": unassessed,
|
||||
"cache_hit": cacheHit, "error": errorText,
|
||||
})
|
||||
return capture.Err()
|
||||
}
|
||||
|
||||
func readableNativeMessages(label string, messages []promptkit.RenderedMessage) string {
|
||||
var text strings.Builder
|
||||
text.WriteString(label + "\n")
|
||||
for _, message := range messages {
|
||||
text.WriteString("\n[" + message.Role + "]\n")
|
||||
text.WriteString(message.Content)
|
||||
text.WriteByte('\n')
|
||||
}
|
||||
return text.String()
|
||||
}
|
||||
|
||||
type scopedEvidence struct {
|
||||
writer contracts.EvidenceWriter
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (scope scopedEvidence) WriteJSON(name string, value any) error {
|
||||
return scope.writer.WriteJSON(path.Join(scope.prefix, name), value)
|
||||
}
|
||||
func (scope scopedEvidence) WriteBytes(name string, value []byte) error {
|
||||
return scope.writer.WriteBytes(path.Join(scope.prefix, name), value)
|
||||
}
|
||||
func scopeEvidence(writer contracts.EvidenceWriter, prefix string) contracts.EvidenceWriter {
|
||||
if writer == nil {
|
||||
return nil
|
||||
}
|
||||
return scopedEvidence{writer: writer, prefix: prefix}
|
||||
}
|
||||
|
||||
func operationExecution(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk) contracts.ExecutionOptions {
|
||||
options := input.Execution
|
||||
writer := options.Capture
|
||||
if writer == nil && input.Debug != nil && input.Debug.Enabled() {
|
||||
writer = input.Debug
|
||||
}
|
||||
prefix := "operations"
|
||||
if origin.StepID != "" {
|
||||
prefix = path.Join(prefix, "steps", fileio.EncodePathComponent(origin.StepID))
|
||||
}
|
||||
prefix = path.Join(prefix, string(origin.Stage))
|
||||
if origin.LaneID != "" {
|
||||
prefix = path.Join(prefix, "lanes", fileio.EncodePathComponent(origin.LaneID))
|
||||
}
|
||||
if chunk != nil {
|
||||
prefix = path.Join(prefix, "chunks", fileio.EncodePathComponent(chunk.ID))
|
||||
}
|
||||
prefix = path.Join(prefix, "modules", fileio.EncodePathComponent(origin.ModuleKey))
|
||||
options.Capture = scopeEvidence(writer, prefix)
|
||||
return options
|
||||
}
|
||||
|
||||
// Native schema/extra-parameter documents are JSON values. Copy their mutable
|
||||
// containers, without serializing large message bodies in OnPrepared.
|
||||
func copyEvidenceJSON(value any) any {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
copy := make(map[string]any, len(value))
|
||||
for key, item := range value {
|
||||
copy[key] = copyEvidenceJSON(item)
|
||||
}
|
||||
return copy
|
||||
case []any:
|
||||
copy := make([]any, len(value))
|
||||
for index, item := range value {
|
||||
copy[index] = copyEvidenceJSON(item)
|
||||
}
|
||||
return copy
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func copyPreparedEvidence(details promptkit.PreparedRun) promptkit.PreparedRun {
|
||||
details.InputHashes = maps.Clone(details.InputHashes)
|
||||
if details.EffectiveModelParams.ExtraParams != nil {
|
||||
details.EffectiveModelParams.ExtraParams = copyEvidenceJSON(details.EffectiveModelParams.ExtraParams).(map[string]any)
|
||||
}
|
||||
details.OutputContract.Schema = copyEvidenceJSON(details.OutputContract.Schema)
|
||||
if details.StructuredOutput != nil {
|
||||
structured := *details.StructuredOutput
|
||||
if structured.JSONSchema != nil {
|
||||
schema := *structured.JSONSchema
|
||||
schema.Schema = copyEvidenceJSON(schema.Schema)
|
||||
structured.JSONSchema = &schema
|
||||
}
|
||||
details.StructuredOutput = &structured
|
||||
}
|
||||
messages := make([]promptkit.RenderedMessage, len(details.Messages))
|
||||
for index, message := range details.Messages {
|
||||
messages[index] = message
|
||||
if message.CacheControl != nil {
|
||||
hint := *message.CacheControl
|
||||
messages[index].CacheControl = &hint
|
||||
}
|
||||
}
|
||||
details.Messages = messages
|
||||
return details
|
||||
}
|
||||
|
||||
func captureFailure(primary, recording error) error {
|
||||
if recording == nil {
|
||||
return primary
|
||||
}
|
||||
return errors.Join(primary, recording)
|
||||
}
|
||||
396
internal/framework/pipeline/native_capture_test.go
Normal file
396
internal/framework/pipeline/native_capture_test.go
Normal file
@@ -0,0 +1,396 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type nativeEvidence struct {
|
||||
mu sync.Mutex
|
||||
files map[string][]byte
|
||||
fail func(string) error
|
||||
}
|
||||
|
||||
func (evidence *nativeEvidence) WriteJSON(name string, value any) error {
|
||||
content, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return evidence.WriteBytes(name, content)
|
||||
}
|
||||
func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error {
|
||||
evidence.mu.Lock()
|
||||
defer evidence.mu.Unlock()
|
||||
if evidence.fail != nil {
|
||||
if err := evidence.fail(name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if evidence.files == nil {
|
||||
evidence.files = make(map[string][]byte)
|
||||
}
|
||||
evidence.files[name] = append([]byte(nil), content...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int {
|
||||
count := 0
|
||||
for name := range evidence.files {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
var group promptkit.ExecutionGroup
|
||||
started, prepared, providerCalls := 0, 0, 0
|
||||
initialExecutions := 0
|
||||
source := strings.Repeat("sk-ordinary-application-text ", 1000)
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
providerCalls++
|
||||
value := 1
|
||||
if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") {
|
||||
initialExecutions++
|
||||
if initialExecutions == 1 {
|
||||
value = 0
|
||||
}
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
||||
}))
|
||||
options := contracts.ExecutionOptions{
|
||||
Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionStarted {
|
||||
started++
|
||||
}
|
||||
}),
|
||||
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ },
|
||||
Capture: evidence,
|
||||
}
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options,
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
request, _ := producerCorrection(ctx, []string{"nested assessment"})
|
||||
request = producerRequest(request, options, true)
|
||||
request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value)))
|
||||
result, err := engine.Run(ctx, request)
|
||||
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
|
||||
return contracts.CandidateAssessment{}, captureErr
|
||||
}
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
if value == 1 {
|
||||
return acceptedProducerAssessment(ctx, value)
|
||||
}
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
initial := nativeProducerInitial()
|
||||
initial.APIKey = "request-credential-must-be-omitted"
|
||||
initial.Inputs["source"] = promptkit.Inline(source)
|
||||
initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
|
||||
details.Messages[0].Content = "changed by existing preparation hook"
|
||||
}
|
||||
value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil)
|
||||
if err != nil || value != 1 {
|
||||
t.Fatalf("result=%d, %v", value, err)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete {
|
||||
t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary)
|
||||
}
|
||||
if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete {
|
||||
t.Fatalf("fabricated known usage: %#v", summary.Usage)
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 {
|
||||
t.Fatalf("missing native results/exchanges: %v", evidence.files)
|
||||
}
|
||||
var initialID string
|
||||
var continuation promptkit.RunResult
|
||||
for name, content := range evidence.files {
|
||||
if strings.Contains(string(content), initial.APIKey) {
|
||||
t.Fatalf("native credential leaked in %s", name)
|
||||
}
|
||||
if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") {
|
||||
t.Fatalf("preparation hook mutated retained evidence: %s", name)
|
||||
}
|
||||
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) {
|
||||
t.Fatal("selected source content was truncated or rewritten")
|
||||
}
|
||||
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") {
|
||||
var result promptkit.RunResult
|
||||
if err := json.Unmarshal(content, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initialID = result.RunID
|
||||
}
|
||||
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") {
|
||||
if err := json.Unmarshal(content, &continuation); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if initialID == "" || continuation.ParentRunID != initialID {
|
||||
t.Fatalf("native parent identity lost: %q %#v", initialID, continuation)
|
||||
}
|
||||
if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") {
|
||||
t.Fatalf("visible rejected exchange history lost: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
operation := numberCacheOperation(store)
|
||||
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := &nativeEvidence{}
|
||||
var group promptkit.ExecutionGroup
|
||||
operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe,
|
||||
OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") },
|
||||
}
|
||||
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
|
||||
t.Fatal("cache hit reassessed")
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||
if err != nil || value != 1 {
|
||||
t.Fatalf("hit=%d, %v", value, err)
|
||||
}
|
||||
if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 {
|
||||
t.Fatal("cache hit fabricated execution/history")
|
||||
}
|
||||
var details promptkit.PreparedRun
|
||||
if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" {
|
||||
t.Fatalf("hit prepared details unavailable: %#v %v", details, err)
|
||||
}
|
||||
if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) {
|
||||
t.Fatal("cache hit not labeled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) {
|
||||
for _, missing := range []bool{false, true} {
|
||||
evidence := &nativeEvidence{}
|
||||
primary := errors.New("provider stopped")
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, primary
|
||||
}))
|
||||
request := nativeProducerInitial()
|
||||
if missing {
|
||||
request.PromptID = "not-present"
|
||||
}
|
||||
request, capture := CaptureNativeRequest(request, evidence)
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if err == nil || result != nil {
|
||||
t.Fatalf("expected result-less native failure: %#v, %v", result, err)
|
||||
}
|
||||
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
|
||||
t.Fatal(captureErr)
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 {
|
||||
t.Fatal("fabricated result/provider body")
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] {
|
||||
t.Fatal("prepared availability wrong")
|
||||
}
|
||||
found := false
|
||||
for name, content := range evidence.files {
|
||||
if strings.HasSuffix(name, "/availability.json") {
|
||||
found = true
|
||||
if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) {
|
||||
t.Fatalf("failure evidence unavailable: %s", content)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("missing failure evidence")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
finished, assessed, fallbackCalls := 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1},
|
||||
Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
finished++
|
||||
if finished == 2 {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}},
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
|
||||
if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" {
|
||||
t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls)
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/result.json") != 2 {
|
||||
t.Fatal("undelivered native response lost")
|
||||
}
|
||||
found := false
|
||||
for name, content := range evidence.files {
|
||||
if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("undelivered response not labeled unassessed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) {
|
||||
for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} {
|
||||
t.Run(boundary, func(t *testing.T) {
|
||||
captureErr := errors.New("evidence write failed")
|
||||
evidence := &nativeEvidence{fail: func(name string) error {
|
||||
if strings.Contains(name, boundary) {
|
||||
return captureErr
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
calls, assessed, fallbackCalls := 0, 0, 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence},
|
||||
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return rejectedProducerAssessment(ctx, value)
|
||||
},
|
||||
}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
|
||||
var recording *contracts.CaptureError
|
||||
if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" {
|
||||
t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls)
|
||||
}
|
||||
if boundary == "request.json" && calls != 0 {
|
||||
t.Fatal("model ran after pre-execution capture failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
var group promptkit.ExecutionGroup
|
||||
var prepared atomic.Int64
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
const operations = 12
|
||||
failures := make(chan error, operations)
|
||||
var workers sync.WaitGroup
|
||||
for index := range operations {
|
||||
workers.Go(func() {
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment,
|
||||
Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)),
|
||||
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) },
|
||||
},
|
||||
}
|
||||
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||
failures <- err
|
||||
})
|
||||
}
|
||||
workers.Wait()
|
||||
close(failures)
|
||||
for err := range failures {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations {
|
||||
t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) {
|
||||
content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000)
|
||||
metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"}
|
||||
envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil)
|
||||
decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64)
|
||||
if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] {
|
||||
t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err)
|
||||
}
|
||||
doc := typedTestDocument()
|
||||
doc.Units[0].Text, doc.Metadata = content, metadata
|
||||
if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 {
|
||||
t.Fatal("parsed source content or ordinary metadata was rewritten")
|
||||
}
|
||||
schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
|
||||
serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}})
|
||||
if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) {
|
||||
t.Fatal("available artifact schema was omitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
|
||||
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil
|
||||
}, nil)
|
||||
if err == nil || calls != 1 {
|
||||
t.Fatalf("missing correction preparation: %v calls=%d", err, calls)
|
||||
}
|
||||
found := false
|
||||
for name, content := range evidence.files {
|
||||
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("constructed correction input lost before preparation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) {
|
||||
primary, recording := errors.New("provider failed"), errors.New("capture failed")
|
||||
evidence := &nativeEvidence{fail: func(name string) error {
|
||||
if strings.HasSuffix(name, "/prepared.json") {
|
||||
return recording
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, primary
|
||||
}))
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil })
|
||||
if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" {
|
||||
t.Fatalf("failure causes lost: %#v %v", outcome, err)
|
||||
}
|
||||
}
|
||||
115
internal/framework/pipeline/operation_capture.go
Normal file
115
internal/framework/pipeline/operation_capture.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func captureOperationInputs(execution contracts.ExecutionOptions, target typedValidationTarget, binding ModuleBinding, incoming []contracts.SerializedArtifact) error {
|
||||
if execution.Capture == nil {
|
||||
return nil
|
||||
}
|
||||
references := make(map[string]any, len(target.references.Slots))
|
||||
for name, slot := range target.references.Slots {
|
||||
items := make([]any, 0, len(slot.Items))
|
||||
for _, item := range slot.Items {
|
||||
items = append(items, map[string]any{"reference": item, "content": debugContentEnvelope(item.Content, item.MediaType, nil, nil)})
|
||||
}
|
||||
references[name] = map[string]any{"declaration": slot.Slot, "items": items}
|
||||
}
|
||||
inputs := make([]debugSerializedOutput, len(incoming))
|
||||
for index, artifact := range incoming {
|
||||
inputs[index] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
|
||||
}
|
||||
var chunk any
|
||||
if target.chunk != nil {
|
||||
chunk = debugSourceChunkEnvelope(*target.chunk)
|
||||
}
|
||||
return execution.RecordEvidence("inputs.json", map[string]any{
|
||||
"source": debugSourceDocumentEnvelope(target.source),
|
||||
"source_input": debugContentEnvelope(target.sourceInput.Content, target.sourceInput.MediaType, nil, nil),
|
||||
"chunk": chunk, "chunks": debugSourceChunkEnvelopes(target.chunks), "references": references,
|
||||
"incoming_artifacts": inputs, "module": binding.Module, "llm_profile": binding.LLMProfile, "metadata": target.metadata,
|
||||
})
|
||||
}
|
||||
|
||||
// Configured options belong to ConfigKit's secret-aware snapshot, not another
|
||||
// direct serialization of runtime configuration inside composition evidence.
|
||||
func captureComposition(resolved ResolvedPipeline) ResolvedPipeline {
|
||||
resolved = cloneResolvedPipeline(resolved)
|
||||
resolved.Input, resolved.Chunk, resolved.Output = captureBinding(resolved.Input), captureBinding(resolved.Chunk), captureBinding(resolved.Output)
|
||||
for stepIndex := range resolved.Steps {
|
||||
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||
lane.Extract, lane.Merge, lane.Normalize = captureBinding(lane.Extract), captureBinding(lane.Merge), captureBinding(lane.Normalize)
|
||||
for index := range lane.Validators {
|
||||
lane.Validators[index] = captureBinding(lane.Validators[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
for index := range resolved.ValidatorChains {
|
||||
for validatorIndex := range resolved.ValidatorChains[index].Validators {
|
||||
resolved.ValidatorChains[index].Validators[validatorIndex].Binding = captureBinding(resolved.ValidatorChains[index].Validators[validatorIndex].Binding)
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func captureBinding(binding ModuleBinding) ModuleBinding {
|
||||
binding.Options = nil
|
||||
for index := range binding.Validators.Validators {
|
||||
binding.Validators.Validators[index] = captureBinding(binding.Validators.Validators[index])
|
||||
}
|
||||
return binding
|
||||
}
|
||||
|
||||
func captureOperationResult(execution contracts.ExecutionOptions, artifact contracts.SerializedArtifact, outcome contracts.ProducerOutcome, failure error) error {
|
||||
if execution.Capture == nil {
|
||||
return nil
|
||||
}
|
||||
var value any
|
||||
if artifact.Content != nil {
|
||||
value = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
|
||||
}
|
||||
errorText := ""
|
||||
if failure != nil {
|
||||
errorText = failure.Error()
|
||||
}
|
||||
return execution.RecordEvidence("outcome.json", map[string]any{"outcome": outcome, "artifact": value, "error": errorText})
|
||||
}
|
||||
|
||||
func captureCacheObservation(writer contracts.EvidenceWriter, observation contracts.CacheObservation) error {
|
||||
if writer == nil {
|
||||
return nil
|
||||
}
|
||||
errorText := ""
|
||||
if observation.Err != nil {
|
||||
errorText = observation.Err.Error()
|
||||
}
|
||||
if err := writer.WriteJSON("cache/"+observation.Decision+".json", map[string]any{
|
||||
"decision": observation.Decision, "message_content_hash": observation.MessageContentHash,
|
||||
"error": errorText,
|
||||
}); err != nil {
|
||||
return &contracts.CaptureError{Cause: err}
|
||||
}
|
||||
if observation.Artifact != nil {
|
||||
if err := writer.WriteBytes("cache/"+observation.Decision+"-artifact.json", observation.Artifact); err != nil {
|
||||
return &contracts.CaptureError{Cause: fmt.Errorf("record cached artifact: %w", err)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func assessmentEvidence(assessment contracts.CandidateAssessment) []any {
|
||||
findings := make([]any, 0, len(assessment.Validators))
|
||||
for _, finding := range assessment.Validators {
|
||||
errorText := ""
|
||||
if finding.Failure != nil {
|
||||
errorText = finding.Failure.Error()
|
||||
}
|
||||
findings = append(findings, map[string]any{"validator": finding.ValidatorName, "decision": finding.Decision,
|
||||
"result": finding.Result, "correction_guidance": finding.Result.CorrectionGuidance, "error": errorText})
|
||||
}
|
||||
return findings
|
||||
}
|
||||
177
internal/framework/pipeline/operation_capture_test.go
Normal file
177
internal/framework/pipeline/operation_capture_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func (*nativeEvidence) Enabled() bool { return true }
|
||||
|
||||
func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) {
|
||||
evidence := &nativeEvidence{}
|
||||
text := strings.Repeat("sk-ordinary-source-content ", 500)
|
||||
reference := strings.Repeat("standalone-reference-content ", 500)
|
||||
doc := typedTestDocument()
|
||||
doc.Units[0].Text = text
|
||||
incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"),
|
||||
chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref},
|
||||
references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}},
|
||||
}
|
||||
binding := Binding("producer")
|
||||
binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"}
|
||||
if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content := string(evidence.files["inputs.json"])
|
||||
for _, value := range []string{text, reference, "chunk material", "incoming-value"} {
|
||||
if !strings.Contains(content, value) {
|
||||
t.Fatalf("missing self-contained input %q", value)
|
||||
}
|
||||
}
|
||||
if strings.Contains(content, "configured-option-must-use-native-inspection") {
|
||||
t.Fatal("runtime configuration bypassed secret-aware inspection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
profile := typedResolutionProfile()
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
|
||||
prepared := prepareRunnerProfile(t, catalog, profile, engine)
|
||||
for _, reuse := range []bool{false, true} {
|
||||
evidence := &nativeEvidence{}
|
||||
var group promptkit.ExecutionGroup
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1})
|
||||
if err != nil || output.FallbackCount != 0 || calls != 1 {
|
||||
t.Fatalf("run: %#v %v, calls=%d", output, err, calls)
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 {
|
||||
t.Fatal("missing chunk/extract/merge/normalization boundaries")
|
||||
}
|
||||
if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] {
|
||||
t.Fatal("cache reuse fabricated a native execution")
|
||||
}
|
||||
for name, content := range evidence.files {
|
||||
if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") {
|
||||
continue
|
||||
}
|
||||
var outcome struct {
|
||||
Artifact struct{ Content debugBinaryEnvelope }
|
||||
}
|
||||
if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) {
|
||||
t.Fatalf("missing codec artifact for %s: %s %v", name, content, err)
|
||||
}
|
||||
}
|
||||
if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 {
|
||||
t.Fatal("reused artifact bytes not retained")
|
||||
}
|
||||
var snapshot promptkit.ExecutionGroupSummary
|
||||
if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete {
|
||||
t.Fatalf("incorrect current-run totals: %#v", snapshot)
|
||||
}
|
||||
if len(evidence.files["composition.json"]) == 0 {
|
||||
t.Fatal("resolved composition unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
probe := newRunnerProbe()
|
||||
primary := errors.New("deterministic operation failed")
|
||||
producer := runnerNotesProducer{name: "notes", probe: probe}
|
||||
if fail {
|
||||
producer.failure = primary
|
||||
}
|
||||
lane := registerRunnerNotes(t, catalog, producer)
|
||||
permissive := contracts.ModePermissive
|
||||
lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe)
|
||||
profile := typedResolutionProfile()
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
|
||||
evidence := &nativeEvidence{}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1})
|
||||
if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) {
|
||||
t.Fatalf("result=%#v %v", output, err)
|
||||
}
|
||||
found := false
|
||||
for name, content := range evidence.files {
|
||||
if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
var result struct {
|
||||
Outcome contracts.ProducerOutcome
|
||||
Error string
|
||||
Artifact any
|
||||
}
|
||||
if err := json.Unmarshal(content, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fail {
|
||||
if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil {
|
||||
t.Fatalf("fabricated failed artifact: %s", content)
|
||||
}
|
||||
} else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil {
|
||||
t.Fatalf("fallback lost qualification: %s", content)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("failed/fallback operation not captured")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) {
|
||||
primary := errors.New("interpretation evidence failed")
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
fallbacks := 0
|
||||
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment}
|
||||
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) {
|
||||
return nil, nil, &contracts.CaptureError{Cause: primary}
|
||||
}, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
||||
if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" {
|
||||
t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) {
|
||||
binding := Binding("configured-module")
|
||||
binding.Options = map[string]any{"token": "configuration-secret"}
|
||||
validator := Binding("configured-validator")
|
||||
validator.Options = map[string]any{"token": "validator-configuration-secret"}
|
||||
binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}}
|
||||
resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}}
|
||||
content, err := json.Marshal(captureComposition(resolved))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") {
|
||||
t.Fatalf("composition lost identities or bypassed native inspection: %s", content)
|
||||
}
|
||||
if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" {
|
||||
t.Fatal("capture mutated configured module options")
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,22 @@ func RunProducer[T any](
|
||||
accepted := false
|
||||
evaluate := func(result *promptkit.RunResult) {
|
||||
candidate, guidance, failure, accepted = nil, nil, nil, false
|
||||
var assessment contracts.CandidateAssessment
|
||||
defer func() {
|
||||
if operation.Execution.Capture == nil || result == nil {
|
||||
return
|
||||
}
|
||||
errorText := ""
|
||||
if failure != nil {
|
||||
errorText = failure.Error()
|
||||
}
|
||||
if err := operation.Execution.RecordEvidence("interpretations/"+result.RunID+".json", map[string]any{
|
||||
"run_id": result.RunID, "usable_candidate": candidate != nil, "accepted": accepted,
|
||||
"correction_guidance": guidance, "assessment": assessmentEvidence(assessment), "error": errorText,
|
||||
}); err != nil {
|
||||
assessmentError = errors.Join(assessmentError, err)
|
||||
}
|
||||
}()
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
failure = canceled
|
||||
return
|
||||
@@ -58,6 +74,10 @@ func RunProducer[T any](
|
||||
return
|
||||
}
|
||||
if failure != nil {
|
||||
var captureErr *contracts.CaptureError
|
||||
if errors.As(failure, &captureErr) {
|
||||
assessmentError = failure
|
||||
}
|
||||
candidate, guidance = nil, nil
|
||||
return
|
||||
}
|
||||
@@ -65,7 +85,8 @@ func RunProducer[T any](
|
||||
failure = errors.New("producer response could not be interpreted as a usable candidate")
|
||||
return
|
||||
}
|
||||
assessment, err := operation.Assess(ctx, *candidate)
|
||||
var err error
|
||||
assessment, err = operation.Assess(ctx, *candidate)
|
||||
if err != nil {
|
||||
assessmentError = err
|
||||
return
|
||||
@@ -85,10 +106,31 @@ func RunProducer[T any](
|
||||
|
||||
request := producerRequest(initial, operation.Execution, true)
|
||||
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
|
||||
request, capture := CaptureNativeRequest(request, scopeEvidence(operation.Execution.Capture, "initial"))
|
||||
if captureErr := capture.Err(); captureErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureErr
|
||||
}
|
||||
var result *promptkit.RunResult
|
||||
var err error
|
||||
var cacheKey string
|
||||
cache := operation.Cache
|
||||
if captureErr := operation.Execution.RecordEvidence("cache/selection.json", map[string]any{"mode": operation.Controls.Cache, "reuse_eligible": operation.ReuseEligible, "store_available": cache != nil && cache.Store != nil}); captureErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureErr
|
||||
}
|
||||
if cache != nil && operation.Execution.Capture != nil {
|
||||
owned := *cache
|
||||
existing := owned.Observe
|
||||
owned.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
|
||||
if err := captureCacheObservation(operation.Execution.Capture, observation); err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
return existing(ctx, observation)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cache = &owned
|
||||
}
|
||||
if cache != nil && cache.Store != nil && operation.ReuseEligible && operation.Controls.Cache != contracts.CacheBypass {
|
||||
if cache.Encode == nil || cache.Decode == nil {
|
||||
return zero, contracts.ProducerOutcome{}, errors.New("producer cache codec must be supplied")
|
||||
@@ -101,15 +143,21 @@ func RunProducer[T any](
|
||||
if err == nil {
|
||||
defer prepared.Discard()
|
||||
details := prepared.Details()
|
||||
if capture != nil {
|
||||
capture.RetainPrepared(copyPreparedEvidence(details))
|
||||
}
|
||||
cacheKey = details.MessageContentHash
|
||||
if captureErr := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "prepared", MessageContentHash: cacheKey, Prepared: &details}); captureErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureErr
|
||||
return zero, contracts.ProducerOutcome{}, captureFailure(captureErr, capture.Finish(nil, captureErr, false, false))
|
||||
}
|
||||
reused, lookupErr := lookupProducerCache(ctx, cache, operation.Controls.Cache, cacheKey)
|
||||
if lookupErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, lookupErr
|
||||
return zero, contracts.ProducerOutcome{}, captureFailure(lookupErr, capture.Finish(nil, lookupErr, false, false))
|
||||
}
|
||||
if reused != nil {
|
||||
if captureErr := capture.Finish(nil, nil, false, true); captureErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureErr
|
||||
}
|
||||
return *reused, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}, nil
|
||||
}
|
||||
result, err = engine.RunPrepared(ctx, prepared)
|
||||
@@ -121,11 +169,14 @@ func RunProducer[T any](
|
||||
decision = "ineligible"
|
||||
}
|
||||
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: decision}); err != nil {
|
||||
return zero, contracts.ProducerOutcome{}, err
|
||||
return zero, contracts.ProducerOutcome{}, captureFailure(err, capture.Finish(nil, err, false, false))
|
||||
}
|
||||
}
|
||||
result, err = engine.Run(ctx, request)
|
||||
}
|
||||
if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureFailure(errors.Join(ctx.Err(), err), captureErr)
|
||||
}
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
@@ -142,11 +193,18 @@ func RunProducer[T any](
|
||||
}
|
||||
|
||||
if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 {
|
||||
var pendingCapture *NativeCapture
|
||||
var recordingErr error
|
||||
coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer,
|
||||
Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
// The first callback has no new result. Its findings already exist.
|
||||
if turn.Result != nil {
|
||||
recordingErr = pendingCapture.Finish(turn.Result, nil, false, false)
|
||||
pendingCapture = nil
|
||||
if recordingErr != nil {
|
||||
return nil, recordingErr
|
||||
}
|
||||
evaluate(turn.Result)
|
||||
}
|
||||
if assessmentError != nil {
|
||||
@@ -158,8 +216,19 @@ func RunProducer[T any](
|
||||
// Once another execution is selected, an earlier candidate is not a
|
||||
// terminal fallback source if construction or native execution fails.
|
||||
candidate = nil
|
||||
correctionPath := ""
|
||||
if operation.Execution.Capture != nil {
|
||||
correctionPath = "corrections/" + turn.Exchange.Details().RunID
|
||||
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/assignment.json", map[string]any{"guidance": guidance})
|
||||
if recordingErr != nil {
|
||||
return nil, recordingErr
|
||||
}
|
||||
}
|
||||
next, buildErr := correct(ctx, guidance)
|
||||
if buildErr != nil {
|
||||
if correctionPath != "" {
|
||||
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/construction-failure.json", map[string]any{"error": buildErr.Error()})
|
||||
}
|
||||
return nil, buildErr
|
||||
}
|
||||
if initial.ProfileID != "" {
|
||||
@@ -169,9 +238,21 @@ func RunProducer[T any](
|
||||
next.SessionID = initial.SessionID
|
||||
}
|
||||
next = producerRequest(next, operation.Execution, false)
|
||||
if operation.Execution.Capture != nil {
|
||||
next, pendingCapture = CaptureNativeRequest(next, scopeEvidence(operation.Execution.Capture, correctionPath))
|
||||
if recordingErr = pendingCapture.Err(); recordingErr != nil {
|
||||
return nil, recordingErr
|
||||
}
|
||||
}
|
||||
return &next, nil
|
||||
},
|
||||
})
|
||||
if pendingCapture != nil {
|
||||
recordingErr = errors.Join(recordingErr, pendingCapture.Finish(coordination.Undelivered, coordinateErr, coordination.Undelivered != nil, false))
|
||||
}
|
||||
if recordingErr != nil {
|
||||
return zero, contracts.ProducerOutcome{}, captureFailure(coordinateErr, recordingErr)
|
||||
}
|
||||
if canceled := ctx.Err(); canceled != nil {
|
||||
return zero, contracts.ProducerOutcome{}, canceled
|
||||
}
|
||||
|
||||
@@ -38,9 +38,9 @@ func lookupProducerCache[T any](ctx context.Context, cache *contracts.ProducerCa
|
||||
return nil, canceled
|
||||
}
|
||||
if err != nil {
|
||||
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err})
|
||||
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err, Artifact: content})
|
||||
}
|
||||
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash}); err != nil {
|
||||
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash, Artifact: content}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &value, nil
|
||||
@@ -59,7 +59,7 @@ func storeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCac
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash}
|
||||
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash, Artifact: content}
|
||||
if err := cache.Store.Store(ctx, mode, hash, content); err != nil {
|
||||
observation.Decision, observation.Err = "store_failed", err
|
||||
}
|
||||
|
||||
@@ -21,6 +21,10 @@ func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM
|
||||
if cause == nil {
|
||||
cause = errors.New("producer did not yield an accepted result")
|
||||
}
|
||||
var captureErr *contracts.CaptureError
|
||||
if errors.As(cause, &captureErr) {
|
||||
return zero, contracts.ProducerOutcome{}, cause
|
||||
}
|
||||
switch mode {
|
||||
case contracts.ModeStrict:
|
||||
return zero, contracts.ProducerOutcome{}, cause
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"path"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type Registries struct {
|
||||
@@ -44,15 +46,18 @@ type RunInput struct {
|
||||
// on a hit. Shared observers must synchronize concurrent calls.
|
||||
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
|
||||
Execution contracts.ExecutionOptions
|
||||
Prepared *PreparedPipeline
|
||||
SourceID string
|
||||
Path string
|
||||
RawInput []byte
|
||||
SessionID string
|
||||
RunID string
|
||||
StartedAt time.Time
|
||||
Metadata map[string]any
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
// Executions is the invocation's native group. The runner enrolls its
|
||||
// requests; callers may use Snapshot without maintaining copied totals.
|
||||
Executions *promptkit.ExecutionGroup
|
||||
Prepared *PreparedPipeline
|
||||
SourceID string
|
||||
Path string
|
||||
RawInput []byte
|
||||
SessionID string
|
||||
RunID string
|
||||
StartedAt time.Time
|
||||
Metadata map[string]any
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
|
||||
Debug DebugRecorder
|
||||
// Workers bounds ready extraction and lane-completion jobs together.
|
||||
@@ -107,12 +112,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}
|
||||
debugRecorder = synchronizedDebugRecorder(debugRecorder)
|
||||
input.Debug = debugRecorder
|
||||
if input.Executions == nil {
|
||||
input.Executions = &promptkit.ExecutionGroup{}
|
||||
}
|
||||
input.Execution.Observer = promptkit.ComposeObservers(input.Execution.Observer, input.Executions.Observe)
|
||||
defer func() {
|
||||
if debugRecorder.Enabled() {
|
||||
if captureErr := debugRecorder.WriteJSON("executions.json", input.Executions.Snapshot()); captureErr != nil {
|
||||
err = errors.Join(err, &contracts.CaptureError{Cause: captureErr})
|
||||
output = failOutput(output)
|
||||
}
|
||||
}
|
||||
}()
|
||||
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
|
||||
if diagnosticErr != nil {
|
||||
return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr)
|
||||
}
|
||||
appendDiagnosticGroups(&output, inputDiagnostics)
|
||||
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
|
||||
if err := writeDebugTimed(debugRecorder, "pipeline.json", debugTimedEnvelope{
|
||||
Stage: "run",
|
||||
StartedAt: startedTime(input.StartedAt),
|
||||
Payload: map[string]any{
|
||||
@@ -123,6 +140,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
|
||||
}
|
||||
if debugRecorder.Enabled() {
|
||||
if err := debugRecorder.WriteJSON("composition.json", captureComposition(input.pipeline)); err != nil {
|
||||
return failOutput(output), &contracts.CaptureError{Cause: err}
|
||||
}
|
||||
}
|
||||
|
||||
adapter := input.Prepared.input
|
||||
var doc *source.SourceDocument
|
||||
@@ -135,8 +157,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
|
||||
Options: redactSensitiveMap(input.pipeline.Input.Options),
|
||||
Metadata: redactSensitiveMap(input.Metadata),
|
||||
Metadata: input.Metadata,
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
|
||||
@@ -184,8 +205,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
"cache_mode": input.pipeline.ChunkControls.Cache,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
|
||||
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
"metadata": input.Metadata,
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
||||
@@ -282,8 +302,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}
|
||||
outputDebugPayload := map[string]any{
|
||||
"normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs),
|
||||
"options": redactSensitiveMap(input.pipeline.Output.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
"metadata": input.Metadata,
|
||||
}
|
||||
if acceptedChunkMap != nil {
|
||||
outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap})
|
||||
|
||||
@@ -21,15 +21,32 @@ type chunkPlanExecution struct {
|
||||
|
||||
// Chunk planning remains serial before lane workers. Only a native chunk
|
||||
// producer uses the accepted-artifact cache; deterministic plans run directly.
|
||||
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
|
||||
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (result chunkPlanExecution, err error) {
|
||||
chunker := input.Prepared.chunker
|
||||
result := chunkPlanExecution{}
|
||||
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
|
||||
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
|
||||
execution := operationExecution(input, origin, nil)
|
||||
defer func() {
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
err = captureFailure(err, execution.RecordEvidence("outcome.json", map[string]any{"outcome": result.outcome, "plan": result.plan, "chunks": debugSourceChunkEnvelopes(result.chunks), "error": errorText}))
|
||||
}()
|
||||
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: execution}
|
||||
if err := captureOperationInputs(execution, target, input.pipeline.Chunk, nil); err != nil {
|
||||
return result, err
|
||||
}
|
||||
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
|
||||
var assessment contracts.CandidateAssessment
|
||||
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
|
||||
candidateNumber := 0
|
||||
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: execution, ReuseEligible: !input.reuseIneligible,
|
||||
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
||||
if execution.Capture != nil {
|
||||
candidateNumber++
|
||||
if err := execution.RecordEvidence(fmt.Sprintf("plans/candidate-%d.json", candidateNumber), plan); err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
}
|
||||
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
||||
if err != nil {
|
||||
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {
|
||||
|
||||
@@ -264,10 +264,10 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
|
||||
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) {
|
||||
state := job.lane
|
||||
lane, typed := state.prepared.resolved, state.prepared.typed
|
||||
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
|
||||
result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
|
||||
if err := ctx.Err(); err != nil {
|
||||
result.err = err
|
||||
return result
|
||||
@@ -283,10 +283,19 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
|
||||
return result
|
||||
}
|
||||
references := operationReferenceSet(input, lane.ExtractReferences)
|
||||
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref}
|
||||
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
||||
var capturedArtifact contracts.SerializedArtifact
|
||||
defer func() {
|
||||
result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err))
|
||||
}()
|
||||
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref}
|
||||
if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil {
|
||||
result.err = err
|
||||
return result
|
||||
}
|
||||
var assessment contracts.CandidateAssessment
|
||||
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
|
||||
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
|
||||
var err error
|
||||
assessment, err = assess(ctx, value)
|
||||
@@ -310,6 +319,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
|
||||
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
||||
return result
|
||||
}
|
||||
capturedArtifact, result.outcome = stored, outcome
|
||||
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
||||
if err != nil {
|
||||
result.err = err
|
||||
@@ -348,7 +358,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.S
|
||||
reuseEligible: state.reuseEligible,
|
||||
}
|
||||
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil {
|
||||
return local, &laneRunError{stage: StageExtract, err: err}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {
|
||||
|
||||
@@ -74,8 +74,7 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *sou
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
|
||||
state := mergeStageResult{}
|
||||
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (state mergeStageResult, err error) {
|
||||
lane, typed := prepared.resolved, prepared.typed
|
||||
references := operationReferenceSet(input, lane.MergeReferences)
|
||||
state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences)
|
||||
@@ -90,10 +89,22 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
|
||||
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
|
||||
var outcome contracts.ProducerOutcome
|
||||
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
|
||||
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
|
||||
if execution.Capture != nil {
|
||||
incoming := make([]contracts.SerializedArtifact, len(extracts.accepted))
|
||||
for index, value := range extracts.accepted {
|
||||
incoming[index] = value.artifact
|
||||
}
|
||||
if err := captureOperationInputs(execution, target, lane.Merge, incoming); err != nil {
|
||||
return state, err
|
||||
}
|
||||
}
|
||||
var assessment contracts.CandidateAssessment
|
||||
assess := typedCandidateAssessor[any](typed.codec, target, prepared.mergeValidators)
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: execution, ReuseEligible: state.reuseEligible,
|
||||
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
|
||||
var err error
|
||||
assessment, err = assess(ctx, value)
|
||||
@@ -130,8 +141,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
|
||||
state := normalizeStageResult{}
|
||||
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (state normalizeStageResult, err error) {
|
||||
lane, typed := prepared.resolved, prepared.typed
|
||||
references := operationReferenceSet(input, lane.NormalizeReferences)
|
||||
state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
|
||||
@@ -142,10 +152,22 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *sou
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
|
||||
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
|
||||
var outcome contracts.ProducerOutcome
|
||||
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
|
||||
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
|
||||
if execution.Capture != nil {
|
||||
incoming, err := serializeArtifact(typed.codec, merged.Value, false)
|
||||
if err != nil {
|
||||
return state, fmt.Errorf("encode normalization input: %w", err)
|
||||
}
|
||||
if err := captureOperationInputs(execution, target, lane.Normalize, []contracts.SerializedArtifact{incoming}); err != nil {
|
||||
return state, err
|
||||
}
|
||||
}
|
||||
var assessment contracts.CandidateAssessment
|
||||
assess := typedCandidateAssessor[any](typed.codec, target, prepared.normalizeValidators)
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
|
||||
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: execution, ReuseEligible: state.reuseEligible,
|
||||
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
|
||||
var err error
|
||||
assessment, err = assess(ctx, value)
|
||||
|
||||
Reference in New Issue
Block a user