Files
notarius/internal/framework/pipeline/runner_chunk_plan.go

247 lines
9.8 KiB
Go

package pipeline
import (
"context"
"fmt"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type chunkPlanExecution struct {
outcome contracts.ProducerOutcome
accepted bool
chunks []source.Chunk
plan *source.ChunkPlan
diagnostics []contracts.DiagnosticGroup
rejection *contracts.RejectedOutput
lookup ChunkPlanDecision
record *ChunkPlanRecord
action string
summary artifacts.ChunkPlanSummary
validation *artifacts.ValidationSummary
}
func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
if mode == "" {
return ChunkCacheBypass
}
parsed, _ := ParseChunkCacheMode(string(mode))
return parsed
}
// Chunk-plan lookup and publication happen serially before lane workers start.
// The runner therefore does not add synchronization around the store.
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
chunker := input.Prepared.chunker
result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}, summary: artifacts.ChunkPlanSummary{Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested"}}
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
var assessment contracts.CandidateAssessment
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: true,
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {
return contracts.CandidateAssessment{}, err
}
assessment = contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "chunk materialization", Decision: contracts.AssessmentFailed, Failure: err}}}
return assessment, nil
}
assessment, err = assessChunks(ctx, chunks)
return assessment, err
},
}
if mode == ChunkCacheAuto {
record, decision, err := input.ChunkPlans.Load(doc.Digest)
if err != nil {
return result, fmt.Errorf("load chunk plan: %w", err)
}
result.lookup = decision
result.summary.LookupStatus, result.summary.LookupReason = chunkPlanLookupStatus(decision.Status), chunkPlanLookupReason(decision.Status)
if decision.Status == ChunkPlanHit {
plan, chunks, err := validateAndMaterializeChunkPlan(doc, record.Plan)
if err == nil {
cachedAssessment, err := assessChunks(ctx, chunks)
if err != nil {
return result, err
}
if cachedAssessment.Accepted() {
if err := result.setCandidate(record, "reused"); err != nil {
return result, err
}
result.plan, result.chunks, result.accepted = &plan, chunks, true
result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.setValidation(nil, err)
return result, err
}
}
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
result.summary.LookupStatus, result.summary.LookupReason = "invalid", chunkPlanLookupReason(ChunkPlanInvalid)
} else if decision.Status != ChunkPlanMissing && decision.Status != ChunkPlanInvalid {
return result, fmt.Errorf("load chunk plan returned unsupported decision status")
}
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
return result, err
}
if err := ctx.Err(); err != nil {
return result, err
}
produced, err := chunker.Plan(ctx, contracts.ChunkRequest{Operation: operation, Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(target.references), LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: metadata})
if err != nil {
return result, fmt.Errorf("chunk source with %q: %w", chunker.Key(), err)
}
plan, outcome, err := completeProducerOperation(ctx, input.pipeline.ChunkExecutionClass, operation, produced.Plan, produced.Outcome, produced.Fallback)
if err != nil {
return result, fmt.Errorf("accept chunk plan with %q: %w", chunker.Key(), err)
}
plan, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
return result, fmt.Errorf("materialize final chunk plan: %w", err)
}
digest, err := source.DigestChunkPlan(plan)
if err != nil {
return result, err
}
producerMetadata, _, err := moduleManifestMetadata(chunker)
if err != nil {
return result, err
}
profile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
profile = input.pipeline.Chunk.LLMProfile
}
record := ChunkPlanRecord{SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: digest, Plan: source.CloneChunkPlan(plan), CreatedAt: time.Now().UTC(),
Producer: ChunkPlanProducer{InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile, References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...), Metadata: producerMetadata},
}
if err := result.setCandidate(record, "generated"); err != nil {
return result, err
}
if mode == ChunkCacheRefresh {
result.action, result.summary.Action = "refreshed", "refreshed"
}
if mode == ChunkCacheBypass {
result.action, result.summary.Action = "bypassed", "bypassed"
}
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
result.setValidation(nil, nil)
if outcome.Disposition == contracts.ProducerFallback {
result.summary.ValidationStatus = "incomplete"
}
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
if err != nil {
return result, err
}
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
result.summary.PublicationStatus = "not_published"
if outcome.ReuseEligible {
record.Diagnostics = contracts.CloneProducerDiagnostics(produced.Diagnostics)
if err := input.ChunkPlans.Save(record); err != nil {
return result, fmt.Errorf("save chunk plan: %w", err)
}
result.summary.PublicationStatus = "published"
}
}
return result, nil
}
func chunkPlanLookupStatus(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "hit"
case ChunkPlanMissing:
return "missing"
case ChunkPlanInvalid:
return "invalid"
default:
return "skipped"
}
}
func chunkPlanLookupReason(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "stored chunk plan is valid"
case ChunkPlanMissing:
return "chunk plan not found"
case ChunkPlanInvalid:
return "stored chunk plan is invalid"
default:
return "chunk plan lookup skipped"
}
}
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
cloned, err := cloneChunkPlanRecord(record)
if err != nil {
return err
}
result.record = &cloned
result.action = action
result.summary.Action = action
result.summary.SourceDigest = record.SourceDigest
result.summary.CandidateDigest = record.PlanDigest
return nil
}
func (result *chunkPlanExecution) setValidation(rejection *contracts.RejectedOutput, err error) {
switch {
case err != nil:
result.summary.ValidationStatus = "error"
case rejection != nil:
result.summary.ValidationStatus = "rejected"
default:
result.summary.ValidationStatus = "approved"
}
}
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
record.Plan = source.CloneChunkPlan(record.Plan)
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
}
record.Producer.Metadata = metadata
record.Diagnostics = contracts.CloneProducerDiagnostics(record.Diagnostics)
return record, nil
}
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
if output == nil {
return nil
}
summary := result.summary
output.ChunkPlan = &summary
if output.Manifest.ChunkPlan == nil {
return nil
}
manifest := output.Manifest.ChunkPlan
manifest.Action = result.action
if result.record == nil {
return nil
}
record := result.record
manifest.SourceDigest = record.SourceDigest
manifest.PlanDigest = record.PlanDigest
manifest.PlanSchemaVersion = record.SchemaVersion
manifest.ProducerInputModule = record.Producer.InputModule
manifest.ProducerModule = record.Producer.ChunkModule
manifest.ProducerLLMProfile = record.Producer.LLMProfile
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
}
manifest.ProducerMetadata = metadata
createdAt := record.CreatedAt
manifest.CreatedAt = &createdAt
return nil
}