265 lines
8.4 KiB
Go
265 lines
8.4 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
type DebugRecorder interface {
|
|
Enabled() bool
|
|
WriteJSON(name string, payload any) error
|
|
WriteBytes(name string, data []byte) error
|
|
}
|
|
|
|
type noopDebugRecorder struct{}
|
|
|
|
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
|
|
|
|
func (noopDebugRecorder) Enabled() bool { return false }
|
|
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
|
|
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
|
|
|
|
type debugTimedEnvelope struct {
|
|
Stage string `json:"stage,omitempty"`
|
|
StepID string `json:"step_id,omitempty"`
|
|
LaneID string `json:"lane_id,omitempty"`
|
|
ModuleKey string `json:"module_key,omitempty"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
CompletedAt time.Time `json:"completed_at"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Payload any `json:"payload,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type debugBinaryEnvelope struct {
|
|
Text string `json:"text,omitempty"`
|
|
ContentBase64 string `json:"content_base64,omitempty"`
|
|
ContentDigest string `json:"content_digest,omitempty"`
|
|
MediaType string `json:"media_type,omitempty"`
|
|
Metadata map[string]any `json:"metadata,omitempty"`
|
|
}
|
|
|
|
type debugSourceInput struct {
|
|
SourceID string `json:"source_id,omitempty"`
|
|
Path string `json:"path,omitempty"`
|
|
Raw debugBinaryEnvelope `json:"raw,omitempty"`
|
|
Metadata map[string]any `json:"metadata,omitempty"`
|
|
}
|
|
|
|
type debugSourceDocument struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
Format string `json:"format,omitempty"`
|
|
Digest string `json:"digest,omitempty"`
|
|
Units []source.SourceUnit `json:"units,omitempty"`
|
|
Metadata map[string]any `json:"metadata,omitempty"`
|
|
}
|
|
|
|
type debugSourceChunk struct {
|
|
ID string `json:"id"`
|
|
SourceID string `json:"source_id"`
|
|
Index int `json:"index"`
|
|
Ref source.SourceRef `json:"ref"`
|
|
Content debugBinaryEnvelope `json:"content"`
|
|
Units []source.SourceUnit `json:"units,omitempty"`
|
|
Metadata map[string]any `json:"metadata,omitempty"`
|
|
Annotations source.ChunkAnnotations `json:"annotations,omitempty"`
|
|
PlanAnnotations source.ChunkAnnotations `json:"plan_annotations,omitempty"`
|
|
}
|
|
|
|
type debugChunkPlan struct {
|
|
SourceDigest string `json:"source_digest,omitempty"`
|
|
Ranges []debugChunkRange `json:"ranges,omitempty"`
|
|
Annotations map[string]any `json:"annotations,omitempty"`
|
|
}
|
|
|
|
type debugChunkRange struct {
|
|
StartUnitID int `json:"start_unit_id"`
|
|
EndUnitID int `json:"end_unit_id"`
|
|
Annotations map[string]any `json:"annotations,omitempty"`
|
|
}
|
|
|
|
type debugSerializedOutput struct {
|
|
StepID string `json:"step_id,omitempty"`
|
|
LaneID string `json:"lane_id"`
|
|
NormalizerKey string `json:"normalizer_key"`
|
|
SourceID string `json:"source_id,omitempty"`
|
|
Kind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
|
Schema contracts.ArtifactSchema `json:"schema"`
|
|
SchemaDigest string `json:"schema_digest"`
|
|
Content debugBinaryEnvelope `json:"content"`
|
|
}
|
|
|
|
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
|
|
if recorder == nil || !recorder.Enabled() {
|
|
return nil
|
|
}
|
|
if envelope.CompletedAt.IsZero() {
|
|
envelope.CompletedAt = time.Now().UTC()
|
|
}
|
|
if envelope.StartedAt.IsZero() {
|
|
envelope.StartedAt = envelope.CompletedAt
|
|
}
|
|
if envelope.DurationMS == 0 {
|
|
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
|
|
}
|
|
return recorder.WriteJSON(name, envelope)
|
|
}
|
|
|
|
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
|
|
text := ""
|
|
if utf8.Valid(content) {
|
|
text = string(content)
|
|
}
|
|
return debugBinaryEnvelope{
|
|
Text: text,
|
|
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
|
ContentDigest: debugContentDigest(content),
|
|
MediaType: mediaType,
|
|
Metadata: metadata,
|
|
}
|
|
}
|
|
|
|
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
|
|
if doc == nil {
|
|
return nil
|
|
}
|
|
return &debugSourceDocument{
|
|
ID: doc.ID,
|
|
Kind: doc.Kind,
|
|
Format: doc.Format,
|
|
Digest: doc.Digest,
|
|
Units: cloneSourceUnitsForDebug(doc.Units),
|
|
Metadata: doc.Metadata,
|
|
}
|
|
}
|
|
|
|
func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
|
|
return debugSourceChunk{
|
|
ID: chunk.ID,
|
|
SourceID: chunk.SourceID,
|
|
Index: chunk.Index,
|
|
Ref: chunk.Ref,
|
|
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
|
Units: cloneSourceUnitsForDebug(chunk.Units),
|
|
Metadata: chunk.Metadata,
|
|
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
|
|
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
|
|
}
|
|
}
|
|
|
|
func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
|
|
if len(units) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make([]source.SourceUnit, len(units))
|
|
for i, unit := range units {
|
|
cloned[i] = source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Ref: unit.Ref,
|
|
Metadata: unit.Metadata,
|
|
}
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
|
|
if len(chunks) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]debugSourceChunk, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
out = append(out, debugSourceChunkEnvelope(chunk))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func debugChunkPlanEnvelope(plan source.ChunkPlan) debugChunkPlan {
|
|
envelope := debugChunkPlan{
|
|
SourceDigest: plan.SourceDigest,
|
|
Ranges: make([]debugChunkRange, len(plan.Ranges)),
|
|
Annotations: debugChunkAnnotations(plan.Annotations),
|
|
}
|
|
for i, chunkRange := range plan.Ranges {
|
|
envelope.Ranges[i] = debugChunkRange{
|
|
StartUnitID: chunkRange.StartUnitID,
|
|
EndUnitID: chunkRange.EndUnitID,
|
|
Annotations: debugChunkAnnotations(chunkRange.Annotations),
|
|
}
|
|
}
|
|
return envelope
|
|
}
|
|
|
|
func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
|
|
if len(annotations) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]any, len(annotations))
|
|
for namespace, raw := range annotations {
|
|
if json.Valid(raw) {
|
|
out[namespace] = append(json.RawMessage(nil), raw...)
|
|
} else {
|
|
out[namespace] = string(raw)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
|
|
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
|
digest := contracts.DigestArtifactSchema(schema)
|
|
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
|
|
content.ContentDigest = debugContentDigest(output.Artifact.Content)
|
|
return debugSerializedOutput{
|
|
StepID: output.StepID, LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
|
|
Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: digest,
|
|
Content: content,
|
|
}
|
|
}
|
|
|
|
func debugSerializedOutputEnvelopes(outputs []contracts.SerializedOutput) []debugSerializedOutput {
|
|
if len(outputs) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]debugSerializedOutput, 0, len(outputs))
|
|
for _, output := range outputs {
|
|
out = append(out, debugSerializedOutputEnvelope(output))
|
|
}
|
|
return out
|
|
}
|
|
|
|
type debugOutputFile struct {
|
|
Name string `json:"name"`
|
|
ContentType string `json:"content_type,omitempty"`
|
|
Content debugBinaryEnvelope `json:"content"`
|
|
}
|
|
|
|
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
|
if len(files) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]debugOutputFile, 0, len(files))
|
|
for _, file := range files {
|
|
out = append(out, debugOutputFile{
|
|
Name: file.Name,
|
|
ContentType: file.ContentType,
|
|
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func debugContentDigest(content []byte) string {
|
|
sum := sha256.Sum256(content)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|