audita/internal/cli/examples_integration_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
	"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
	ck "go.maximumdirect.net/configkit"
)

func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
	_, file, _, ok := runtime.Caller(0)
	if !ok {
		t.Fatal("cannot locate examples")
	}
	root := filepath.Join(filepath.Dir(file), "..", "..", "examples")
	t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key")
	for _, tc := range []struct{ dir, main, overlay string }{
		{"local", "config.yml", ""}, {"service", "config.yml", "testing"},
		{"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""},
	} {
		t.Run(tc.dir+"/"+tc.main, func(t *testing.T) {
			main := filepath.Join(root, tc.dir, tc.main)
			loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay)
			if err != nil {
				t.Fatal(err)
			}
			loaded.Config.Reporting.Enabled = false
			var calls atomic.Int32
			var mu sync.Mutex
			var models, prompts []string
			transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
				calls.Add(1)
				var request struct {
					Model    string `json:"model"`
					Messages []struct {
						Content string `json:"content"`
					} `json:"messages"`
				}
				if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
					return nil, err
				}
				var content strings.Builder
				for _, message := range request.Messages {
					content.WriteString(message.Content)
				}
				mu.Lock()
				models = append(models, request.Model)
				prompts = append(prompts, content.String())
				mu.Unlock()
				return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil
			})
			engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
			if err != nil {
				t.Fatal(err)
			}
			if tc.dir == "prompt-edit" {
				prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
				if err != nil || !strings.Contains(prompt.Messages[0].Content, "clearly supported grammatical errors") {
					t.Fatalf("operator prompt: %+v %v", prompt, err)
				}
			}
			raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json"))
			if err != nil {
				t.Fatal(err)
			}
			source, err := schema.ParseSourceTranscriptJSON(raw)
			if err != nil {
				t.Fatal(err)
			}
			inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source}
			prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{})
			if err != nil {
				t.Fatal(err)
			}
			defer prepared.Prepared.Output.Close()
			if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil {
				t.Fatal(err)
			}
			if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
				t.Fatalf("provider calls=%d", calls.Load())
			}
			if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") {
				t.Fatalf("overlay models=%v", models)
			}
			if tc.dir == "campaigns" && tc.main == "first.yml" {
				if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) {
					t.Fatalf("campaign glossary selections: %v", prompts)
				}
			}
			if tc.dir == "campaigns" && tc.main == "second.yml" {
				if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) {
					t.Fatalf("second campaign glossary selection: %v", prompts)
				}
			}
		})
	}
}

func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) {
	_, file, _, ok := runtime.Caller(0)
	if !ok {
		t.Fatal("cannot locate examples")
	}
	source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit")
	dir := t.TempDir()
	for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} {
		raw, err := os.ReadFile(filepath.Join(source, name))
		if err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil {
			t.Fatal(err)
		}
	}
	inspect := func() string {
		loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
		if err != nil {
			t.Fatal(err)
		}
		engine, err := llm.NewRuntime(context.Background(), loaded, nil)
		if err != nil {
			t.Fatal(err)
		}
		prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
		if err != nil {
			t.Fatal(err)
		}
		return prompt.Messages[0].Content
	}
	before := inspect()
	filePath := filepath.Join(dir, "prompts.yml")
	raw, err := os.ReadFile(filePath)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil {
		t.Fatal(err)
	}
	after := inspect()
	if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") {
		t.Fatalf("before=%q after=%q", before, after)
	}
}