package cli import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "reflect" "strings" "sync" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" ck "go.maximumdirect.net/configkit" ) type controlledTransport func(*http.Request) (*http.Response, error) func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) { var mu sync.Mutex var models []string var prompts []string transport := controlledTransport(func(r *http.Request) (*http.Response, error) { var request struct { Model string `json:"model"` Messages []struct { Content string `json:"content"` } `json:"messages"` } if err := json.NewDecoder(r.Body).Decode(&request); err != nil { return nil, err } var contents string for _, message := range request.Messages { contents += message.Content } mu.Lock() models = append(models, request.Model) prompts = append(prompts, contents) mu.Unlock() result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}` switch request.Model { case "model-one": result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}` case "model-two": result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}` case "model-three": result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast","confidence":1}]}` } encoded, _ := json.Marshal(result) body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded) return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil }) dir := t.TempDir() files := map[string]string{ "config.yml": `version: 3 reporting: {enabled: false} glossary_collections: party: entries: [{id: alice, name: Alice, category: person}] spells: entries: [{id: shield, name: Shield, category: spell}] protected_terms: [party, spells] llm: {validation_profile: review} pipeline: stages: - module: glossary glossaries: [party] llm: {proposal_profile: one} - module: glossary glossaries: [spells] llm: {proposal_profile: two} - module: grammar glossaries: [party, spells] llm: {proposal_profile: three} `, "prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"), "transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`, } for name, value := range files { if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil { t.Fatal(err) } } loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"])) if err != nil { t.Fatal(err) } inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source} prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{}) if err != nil { t.Fatal(err) } defer prepared.Prepared.Output.Close() result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}) if err != nil { t.Fatal(err) } if result.Transcript.Segments[0].Text != "Alice cast Shield." { t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models) } if len(prompts) > 3 && !strings.Contains(prompts[3], "Alice casts Sheeld.") { t.Fatalf("second proposal missed earlier edit: %s", prompts[3]) } if !reflect.DeepEqual(models, []string{"model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) { t.Fatalf("models=%v", models) } } func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) { dir := t.TempDir() files := map[string]string{ "config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar}]\n", "prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n", "transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`, } for name, value := range files { if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil { t.Fatal(err) } } loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } transport := controlledTransport(func(r *http.Request) (*http.Response, error) { return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil }) runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"])) if err != nil { t.Fatal(err) } inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source} prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{}) if err != nil { t.Fatal(err) } defer prepared.Prepared.Output.Close() result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}) if err != nil { t.Fatal(err) } if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" { t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors) } }