Verify native workflows and explain identity mismatches
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
@@ -16,16 +17,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
func TestSelectedPartyCatalogDrivesBoundedAnalysis(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `whisperx:
|
||||
const secret = "synthetic-family-workflow-credential"
|
||||
t.Setenv("NARRATIO_WORKFLOW_TEST_KEY", secret)
|
||||
mustWriteTestFile(t, pipelinePath, "config:\n imports:\n - {path: family.yml, required: true}\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "overlays", "testing.yml"), "scriptorium:\n artifact_families:\n character_note:\n prompt_id: note-testing\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "family.yml"), `whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
workspace:
|
||||
root: `+filepath.Join(dir, "work")+`
|
||||
scriptorium:
|
||||
credentials: [NARRATIO_WORKFLOW_TEST_KEY]
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: false
|
||||
@@ -50,7 +57,8 @@ scriptorium:
|
||||
mustWriteTestFile(t, partyPath, "schema_version: narratio.party.v1\ncharacters:\n ivy:\n player: {name: Selected Player}\n character: {name: Ivy, classes: [{name: ranger}]}\n")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
mustWriteTestFile(t, sessionPath, "session_id: session\ninputs: {audio_dir: audio}\n")
|
||||
pipeline, err := config.LoadPipeline(pipelinePath)
|
||||
overlay := "testing"
|
||||
pipeline, err := config.LoadPipelineWithOptions(pipelinePath, config.PipelineLoadOptions{Overlay: &overlay})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -70,6 +78,45 @@ scriptorium:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogSelection := []string{"--config", pipelinePath, "--overlay", overlay, "--campaign-file", campaignPath, "--party-file", partyPath, "--secrets-dir", t.TempDir(), "--format", "json"}
|
||||
common := append(append([]string{}, catalogSelection...), "--session-file", sessionPath)
|
||||
for _, scope := range []string{"app", "session", "party"} {
|
||||
output := nativeInspectionForTest(t, append([]string{"config", "show", "--scope", scope}, common...))
|
||||
if strings.Contains(output, secret) {
|
||||
t.Fatal("native inspection exposed credentials")
|
||||
}
|
||||
if scope == "app" && (!strings.Contains(output, "artifact_families") || strings.Contains(output, "character_meta_ivy")) {
|
||||
t.Fatal("native app inspection substituted derived configuration")
|
||||
}
|
||||
if scope == "session" {
|
||||
var effective map[string]any
|
||||
if err := json.Unmarshal([]byte(output), &effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if effective["campaign"] != "campaign" || effective["session_id"] != "session" {
|
||||
t.Fatalf("inspection lost resolved session defaults: %#v", effective)
|
||||
}
|
||||
}
|
||||
if scope == "party" && !strings.Contains(output, "Selected Player") {
|
||||
t.Fatal("party inspection used a different roster")
|
||||
}
|
||||
}
|
||||
diff := nativeInspectionForTest(t, []string{"config", "diff", "--config", pipelinePath, "--against-config", pipelinePath, "--against-overlay", overlay, "--details", "--format", "json"})
|
||||
if !strings.Contains(diff, "/scriptorium/artifact_families/character_note/prompt_id") || strings.Contains(diff, secret) {
|
||||
t.Fatalf("unexpected independent comparison: %s", diff)
|
||||
}
|
||||
catalog := nativeInspectionForTest(t, append([]string{"artifacts", "list"}, catalogSelection...))
|
||||
if !strings.Contains(catalog, "character_note_ivy") {
|
||||
t.Fatal("catalog omitted selected member")
|
||||
}
|
||||
selected := []string{"--stage", "analyze", "--artifacts", "character_note"}
|
||||
planned := nativeInspectionForTest(t, append(append([]string{"session", "plan", "session"}, common...), "--stage", "prepare"))
|
||||
if !strings.Contains(planned, "prepare") || strings.Contains(planned, secret) {
|
||||
t.Fatal("initial plan lost selection or exposed credentials")
|
||||
}
|
||||
if _, err := os.Stat(cfg.Pipeline.Workspace.Root); !os.IsNotExist(err) {
|
||||
t.Fatalf("inspection or planning created workspace: %v", err)
|
||||
}
|
||||
plan, err := BuildBoundedPlan("analyze", "analyze")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -93,16 +140,71 @@ scriptorium:
|
||||
}
|
||||
seed.Inputs = append(seed.Inputs, manifest.InputRecord{Kind: descriptor.ManifestKind, Path: playersPath, Checksum: checksum})
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
runner := &scriptorium.FakeRunner{}
|
||||
summary, err := executePlan(context.Background(), cfg, plan, RunOptions{
|
||||
SelectedArtifacts: []string{"character_note"},
|
||||
Env: &Env{Scriptorium: runner},
|
||||
persisted, err := os.ReadFile(manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planned = nativeInspectionForTest(t, append(append([]string{"session", "plan", "session"}, common...), selected...))
|
||||
if !strings.Contains(planned, "character_meta_ivy") || strings.Contains(planned, secret) {
|
||||
t.Fatal("plan lost prerequisites or exposed credentials")
|
||||
}
|
||||
afterPlan, err := os.ReadFile(manifestPathFor(cfg))
|
||||
if err != nil || !bytes.Equal(persisted, afterPlan) {
|
||||
t.Fatal("planning changed durable state")
|
||||
}
|
||||
failure := errors.New("synthetic later artifact failure")
|
||||
runner := &familyWorkflowRunner{failure: failure}
|
||||
var captured *config.Config
|
||||
application, err := newApplication(commandDependencies{}, func(ctx context.Context, acquired *config.Config, selected BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
captured = acquired
|
||||
before, err := json.Marshal(acquired.Pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options.Env = &Env{Scriptorium: runner}
|
||||
summary, runErr := executePlan(ctx, acquired, selected, options)
|
||||
after, err := json.Marshal(acquired.Pipeline)
|
||||
if err != nil || !bytes.Equal(before, after) {
|
||||
t.Fatal("native execution mutated completed configuration")
|
||||
}
|
||||
return summary, runErr
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(summary.Executed, []string{"analyze"}) || len(runner.RunRequests) != 2 || runner.RunRequests[0].PromptID != "meta" || runner.RunRequests[1].PromptID != "note" {
|
||||
t.Fatalf("bounded family execution: summary=%+v requests=%+v", summary, runner.RunRequests)
|
||||
args := append(append([]string{"run", "session"}, common...), selected...)
|
||||
var output, diagnostics bytes.Buffer
|
||||
outcome := application.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, failure) {
|
||||
t.Fatalf("failed outcome=%#v diagnostics=%s", outcome, diagnostics.String())
|
||||
}
|
||||
partial := decodeNativeReport[runReport](t, output.Bytes())
|
||||
if partial.Complete || len(partial.StageResults) != 1 || partial.StageResults[0].Outcome != "failed" || len(partial.AvailableArtifacts) == 0 {
|
||||
t.Fatalf("partial report=%#v", partial)
|
||||
}
|
||||
failed, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failed.Stages["analyze"].AnalyzeArtifacts["character_meta_ivy"].Status != manifest.AnalyzeArtifactCurrent {
|
||||
t.Fatal("earlier family checkpoint was lost")
|
||||
}
|
||||
if strings.Contains(output.String()+diagnostics.String(), secret) {
|
||||
t.Fatal("failure report exposed credentials")
|
||||
}
|
||||
runner.failure = nil
|
||||
output.Reset()
|
||||
diagnostics.Reset()
|
||||
outcome = application.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("retry status=%d diagnostics=%s", outcome.Status, diagnostics.String())
|
||||
}
|
||||
summary := decodeNativeReport[runReport](t, output.Bytes())
|
||||
if !summary.Complete || !reflect.DeepEqual(summary.Executed, []string{"analyze"}) || len(runner.RunRequests) != 2 || runner.RunRequests[0].PromptID != "meta" || runner.RunRequests[1].PromptID != "note-testing" {
|
||||
t.Fatalf("bounded family retry: summary=%+v requests=%+v", summary, runner.RunRequests)
|
||||
}
|
||||
if captured == nil || len(captured.Pipeline.Scriptorium.Artifacts) != 0 || strings.Contains(output.String()+diagnostics.String(), secret) {
|
||||
t.Fatal("retry lost the configuration/report boundary")
|
||||
}
|
||||
players, err := os.ReadFile(runner.RunRequests[0].InputPaths["players"])
|
||||
if err != nil || !strings.Contains(string(players), "Selected Player") || runner.RunRequests[1].Vars["character_name"] != "Ivy" {
|
||||
@@ -126,3 +228,17 @@ scriptorium:
|
||||
t.Fatal("execution changed completed pipeline configuration")
|
||||
}
|
||||
}
|
||||
|
||||
// familyWorkflowRunner replaces only the downstream process, retaining real
|
||||
// acquisition, catalog derivation, execution, persistence, and reporting.
|
||||
type familyWorkflowRunner struct {
|
||||
scriptorium.FakeRunner
|
||||
failure error
|
||||
}
|
||||
|
||||
func (r *familyWorkflowRunner) RunArtifact(ctx context.Context, request scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
|
||||
if request.PromptID == "note-testing" && r.failure != nil {
|
||||
return scriptorium.ArtifactResult{}, r.failure
|
||||
}
|
||||
return r.FakeRunner.RunArtifact(ctx, request)
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
|
||||
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), SessionFile: suppliedSelection(sessionPath), Session: suppliedSelection("2026-05-03")}, deps)
|
||||
resolved, err := loaded.Domain(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
t.Fatalf("configuration acquisition error = %v", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
|
||||
|
||||
@@ -77,6 +77,10 @@ func configurationCommand(deps commandDependencies) commandkit.Command {
|
||||
scopes[index].Acquire = func(ctx context.Context, request ckconfig.Request, input configurationSelection, state *configurationInspectionState) (commandkit.Acquired[ckconfig.View], error) {
|
||||
result, err := acquire(ctx, request, input, state)
|
||||
result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...)
|
||||
var identity *config.SessionIdentityError
|
||||
if errors.As(err, &identity) {
|
||||
result.Explanation.Message = identity.Error()
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
compare := scopes[index].Compare
|
||||
@@ -85,6 +89,10 @@ func configurationCommand(deps commandDependencies) commandkit.Command {
|
||||
separate.independent = true
|
||||
result, err := compare(ctx, request, &separate)
|
||||
result.Diagnostics = append(result.Diagnostics, state.bootstrapDiagnostics()...)
|
||||
var identity *config.SessionIdentityError
|
||||
if errors.As(err, &identity) {
|
||||
result.Explanation.Message = identity.Error()
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -167,3 +168,25 @@ func TestConfigurationAggregateDoesNotOmitUnavailableContext(t *testing.T) {
|
||||
t.Fatalf("missing aggregate scope report: %s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionExplainsIdentityMismatches(t *testing.T) {
|
||||
_, campaign, session := writeValidConfigFiles(t, t.TempDir())
|
||||
for _, args := range [][]string{
|
||||
{"config", "show", "--scope", "campaign", "--campaign-file", campaign, "--campaign", "other-campaign"},
|
||||
{"config", "show", "--scope", "session", "--campaign-file", campaign, "--session-file", session, "--session", "other-session"},
|
||||
{"config", "diff", "--scope", "campaign", "--campaign-file", campaign, "--against-campaign-file", campaign, "--against-campaign", "other-campaign"},
|
||||
} {
|
||||
var output, diagnostics bytes.Buffer
|
||||
result := Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: &diagnostics})
|
||||
var identity *config.SessionIdentityError
|
||||
if result.Status == 0 || !errors.As(result.Cause, &identity) {
|
||||
t.Fatalf("%v: identity cause missing: %#v", args, result)
|
||||
}
|
||||
if output.Len() != 0 || !strings.Contains(diagnostics.String(), identity.Field) || !strings.Contains(diagnostics.String(), identity.Expected) || !strings.Contains(diagnostics.String(), identity.Actual) {
|
||||
t.Fatalf("%v: assertion not explained: %s", args, diagnostics.String())
|
||||
}
|
||||
if identity.Field == "campaign_id" && strings.Contains(diagnostics.String(), "session file") {
|
||||
t.Fatalf("campaign assertion mislabeled: %s", diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ inputs:
|
||||
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps)
|
||||
resolved, err := loaded.Domain(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
t.Fatalf("configuration acquisition error = %v", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
if resolved.Party.Canonical == nil {
|
||||
@@ -222,7 +222,7 @@ inputs:
|
||||
loaded := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection(pipelinePath), CampaignFile: suppliedSelection(campaignPath), Session: suppliedSelection("2026-05-03")}, deps)
|
||||
resolved, err := loaded.Domain(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
t.Fatalf("configuration acquisition error = %v", err)
|
||||
}
|
||||
downloadedPath := resolved.SessionSource.LocalPath
|
||||
if err := loaded.Close(); err != nil {
|
||||
|
||||
@@ -230,7 +230,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRestoreNonDryRunConflictDoesNotexecuteForTest(t *testing.T) {
|
||||
func TestExecuteRestoreNonDryRunConflictDoesNotWrite(t *testing.T) {
|
||||
deps := restoreCommandDependencies(&storage.FakeBackend{})
|
||||
deps.discoverRemoteCurrentState = func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error) {
|
||||
return &RemoteCurrentState{
|
||||
|
||||
@@ -82,10 +82,14 @@ type SessionIdentityError struct {
|
||||
}
|
||||
|
||||
func (e *SessionIdentityError) Error() string {
|
||||
if e.Actual == "" {
|
||||
return fmt.Sprintf("load session config: session file %q: %s is required when %q is selected", e.Source, e.Field, e.Expected)
|
||||
sourceKind := "session"
|
||||
if e.Field == "campaign_id" {
|
||||
sourceKind = "campaign"
|
||||
}
|
||||
return fmt.Sprintf("load session config: session file %q: %s mismatch: selected %q does not match %q", e.Source, e.Field, e.Expected, e.Actual)
|
||||
if e.Actual == "" {
|
||||
return fmt.Sprintf("load %s config: %s file %q: %s is required when %q is selected", sourceKind, sourceKind, e.Source, e.Field, e.Expected)
|
||||
}
|
||||
return fmt.Sprintf("load %s config: %s file %q: %s mismatch: selected %q does not match %q", sourceKind, sourceKind, e.Source, e.Field, e.Expected, e.Actual)
|
||||
}
|
||||
|
||||
func validateSessionIdentity(label string, cfg *SessionConfig, opts SessionLoadOptions) error {
|
||||
|
||||
Reference in New Issue
Block a user