Compare commits
115 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2be7bbc015 | |||
| 64e5d0d275 | |||
| ca36c3e319 | |||
| 6d34602f0a | |||
| 891c204e8f | |||
| 73ef2c6083 | |||
| 04e0c2f913 | |||
| 6b724fd3cc | |||
| 8c93f728c0 | |||
| 5d90478f70 | |||
| 45b42a2689 | |||
| 8267eba6a5 | |||
| bcce4729e1 | |||
| d3a9edb29d | |||
| d3fbddce11 | |||
| e029730c79 | |||
| aa8073cab1 | |||
| f4c638313e | |||
| 735dfc1834 | |||
| d927c61199 | |||
| a5003a1e52 | |||
| 9fd36001b4 | |||
| 00848203d7 | |||
| eb45d0b623 | |||
| bb81354e21 | |||
| 93a080059d | |||
| b036336e2a | |||
| 4f525e07cc | |||
| d91bb658c9 | |||
| 8946a82407 | |||
| 791deca3cd | |||
| c86f4fa416 | |||
| 2140314d40 | |||
| 71a9b1f560 | |||
| a9b7c37a7e | |||
| c16d61e091 | |||
| 8ad27e8368 | |||
| 1fd95abe95 | |||
| bb63c2cb10 | |||
| 7710101da0 | |||
| 50940083c8 | |||
| 8a2a19ccd3 | |||
| bbac312b32 | |||
| 56d39fca80 | |||
| 9a52c8c246 | |||
| a0e45c8f92 | |||
| bae96e5dd5 | |||
| 803effc6ea | |||
| 45e5b381f5 | |||
| a8eb5aea4d | |||
| af0f9ac960 | |||
| 3eacb33eb0 | |||
| 88e166e3b1 | |||
| fa4f891689 | |||
| 2844ce7e6a | |||
| ba1d5d2367 | |||
| 926fa26a31 | |||
| 7efa07ee9d | |||
| 89de2a7014 | |||
| d42e2095d7 | |||
| 0a21ee2b50 | |||
| 199c1d1dbe | |||
| 852e79405f | |||
| 8e1e073731 | |||
| 8cc298d092 | |||
| 50257ac554 | |||
| 497a1a18c5 | |||
| 7c64592cab | |||
| b500b855c6 | |||
| 40420a217a | |||
| 0c7622375f | |||
| 4920393204 | |||
| 866e4b9b23 | |||
| 547be2d9ab | |||
| f1fced498a | |||
| 97f6f48098 | |||
| 267aca4b76 | |||
| d539c06f45 | |||
| 5c69377563 | |||
| 098401f674 | |||
| f016471880 | |||
| 7e9bf6f149 | |||
| 77995b7a2c | |||
| 1b76a426cc | |||
| 607fefac47 | |||
| 6908dcd206 | |||
| 42930d4921 | |||
| 64786140e8 | |||
| af6f62e14b | |||
| 167d91131e | |||
| 4019f57eaa | |||
| 760a7cecc0 | |||
| 943152d1d5 | |||
| c94086048a | |||
| d2c0e22a9a | |||
| 0b3734d352 | |||
| b7bad87659 | |||
| c46a689e29 | |||
| 9dba2d506a | |||
| 68e16f6aac | |||
| 1f2beef08c | |||
| 32bac58b5e | |||
| 90d82b5b5f | |||
| a9dba890ba | |||
| 04f9f16e27 | |||
| 893f66b34b | |||
| 98ffb67432 | |||
| ec8dffc06e | |||
| d4d510a520 | |||
| ac86c77cfe | |||
| 259f728f20 | |||
| 46765bd326 | |||
| cbcb1b3778 | |||
| 39052bf47e | |||
| 7cd5edd650 |
@@ -1,6 +1,6 @@
|
|||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- GO_IMAGE: golang:1.26.8-bookworm
|
- GO_IMAGE: golang:1.27.3-bookworm
|
||||||
|
|
||||||
when:
|
when:
|
||||||
- event: tag
|
- event: tag
|
||||||
@@ -20,7 +20,7 @@ steps:
|
|||||||
git check-ref-format "refs/tags/$${CI_COMMIT_TAG}"
|
git check-ref-format "refs/tags/$${CI_COMMIT_TAG}"
|
||||||
git fetch origin "refs/tags/$${CI_COMMIT_TAG}:refs/tags/$${CI_COMMIT_TAG}"
|
git fetch origin "refs/tags/$${CI_COMMIT_TAG}:refs/tags/$${CI_COMMIT_TAG}"
|
||||||
tag_commit=$(git rev-parse "refs/tags/$${CI_COMMIT_TAG}^{commit}")
|
tag_commit=$(git rev-parse "refs/tags/$${CI_COMMIT_TAG}^{commit}")
|
||||||
expected_commit=$(printf '%s' "$${CI_COMMIT_SHA}" | tr '[:upper:]' '[:lower:]')
|
expected_commit=$(git rev-parse --verify "$${CI_COMMIT_SHA}^{commit}")
|
||||||
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
git checkout --detach "$${tag_commit}"
|
git checkout --detach "$${tag_commit}"
|
||||||
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
@@ -28,23 +28,24 @@ steps:
|
|||||||
|
|
||||||
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
|
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
|
||||||
trap 'rm -rf "$${bootstrap_dir}"' EXIT
|
trap 'rm -rf "$${bootstrap_dir}"' EXIT
|
||||||
cp go.mod "$${bootstrap_dir}/go.mod"
|
cp go.mod "$${bootstrap_dir}/releasekit.mod"
|
||||||
if [ -f go.sum ]; then
|
if [ -f go.sum ]; then
|
||||||
cp go.sum "$${bootstrap_dir}/go.sum"
|
cp go.sum "$${bootstrap_dir}/releasekit.sum"
|
||||||
else
|
else
|
||||||
: > "$${bootstrap_dir}/go.sum"
|
: > "$${bootstrap_dir}/releasekit.sum"
|
||||||
fi
|
fi
|
||||||
cp "$${bootstrap_dir}/go.mod" "$${bootstrap_dir}/releasekit.mod"
|
|
||||||
cp "$${bootstrap_dir}/go.sum" "$${bootstrap_dir}/releasekit.sum"
|
|
||||||
|
|
||||||
export GOWORK=off
|
export GOWORK=off
|
||||||
export GOTOOLCHAIN=local
|
export GOTOOLCHAIN=local
|
||||||
releasekit_module=gitea.maximumdirect.net/eric/ReleaseKit
|
releasekit_host_os=$(go env GOHOSTOS) || exit $?
|
||||||
|
releasekit_host_arch=$(go env GOHOSTARCH) || exit $?
|
||||||
|
export GOOS="$${releasekit_host_os}" GOARCH="$${releasekit_host_arch}"
|
||||||
|
releasekit_module=go.maximumdirect.net/releasekit
|
||||||
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -m "$${releasekit_module}"
|
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -m "$${releasekit_module}"
|
||||||
go build -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" \
|
go mod edit -modfile="$${bootstrap_dir}/releasekit.mod" -tool="$${releasekit_module}/cmd/releasekit"
|
||||||
-o "$${bootstrap_dir}/releasekit" "$${releasekit_module}/cmd/releasekit"
|
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -deps "$${releasekit_module}/cmd/releasekit" > /dev/null
|
||||||
"$${bootstrap_dir}/releasekit" ci-deliver \
|
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
|
||||||
--repo "$${CI_WORKSPACE}" \
|
--repo "$${CI_WORKSPACE}" \
|
||||||
--prepared \
|
--prepared \
|
||||||
--commit "$${CI_COMMIT_SHA}" \
|
--commit "$${expected_commit}" \
|
||||||
"$${CI_COMMIT_TAG}"
|
"$${CI_COMMIT_TAG}"
|
||||||
|
|||||||
49
README.md
49
README.md
@@ -1,55 +1,34 @@
|
|||||||
# Audita
|
# Audita
|
||||||
|
|
||||||
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
|
Audita polishes transcript JSON with glossary-aware and LLM-backed correction
|
||||||
modules. It validates proposed corrections and produces corrected text plus
|
modules. It validates proposed edits and produces corrected text and reports for
|
||||||
reports for review and subprocess integration.
|
review or subprocess integration.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
From a checkout, build with Go 1.26.0 or newer:
|
Build with Go 1.27.0 or newer:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go build -o ./bin/audita ./cmd/audita
|
go build -o ./bin/audita ./cmd/audita
|
||||||
```
|
```
|
||||||
|
|
||||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
Edit the synthetic endpoint and model in [examples/local/prompts.yml](examples/local/prompts.yml)
|
||||||
and access to a model you select. The [minimal config](examples/minimal-config.yml)
|
for a reachable OpenAI-compatible service. From `examples/local`, run the built
|
||||||
targets an unauthenticated local endpoint; edit its endpoint and
|
executable (adjust the executable path if needed):
|
||||||
[local profiles](examples/profiles/local/) to match your service and models. The
|
|
||||||
[service example](examples/production-config.yml) demonstrates a named credential
|
|
||||||
reference and a shared backend. Set its referenced environment variable before
|
|
||||||
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
|
|
||||||
role profiles, but model cost, speed, and quality depend on your provider choices.
|
|
||||||
|
|
||||||
Check configuration and profile selection offline:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./bin/audita config validate --config examples/minimal-config.yml
|
../../bin/audita config validate --config config.yml
|
||||||
./bin/audita config print-effective --config examples/minimal-config.yml --view profiles
|
../../bin/audita process --config config.yml --output corrected.json transcript.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the included small input with the minimal config:
|
Processing makes model requests. Reports are written beside the output by default;
|
||||||
|
recoverable model errors can appear in a completed run. The [example guide](examples/README.md)
|
||||||
```sh
|
includes service, overlay, prompt editing, preview, and shared glossary projects.
|
||||||
./bin/audita process examples/tiny-transcript.json \
|
See the [v3 migration guide](docs/migrations/v3.md) for the configuration cutover.
|
||||||
--config examples/minimal-config.yml \
|
|
||||||
--glossary examples/tiny-glossary.yaml \
|
|
||||||
--output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
This makes LLM requests. Explicit setting flags override the selected file and
|
|
||||||
optional overlay; general environment setting overrides are no longer read.
|
|
||||||
By default the command writes `errors.yml`, `recoveries.yml`, and `report.md`
|
|
||||||
beside `corrected.json`. Use `--reporting=false` to suppress those files, or
|
|
||||||
`--debug` to retain detailed run artifacts under the configured debug work
|
|
||||||
directory. Model failures affecting some operations can still yield a completed
|
|
||||||
transcript with applied corrections; the summary and reports identify them.
|
|
||||||
See the [v2 migration guide](docs/migrations/v2.md) for old configurations.
|
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md)
|
- [CLI reference](docs/cli.md) and [configuration](docs/config.md)
|
||||||
- [Configuration and maintained examples](docs/config.md)
|
|
||||||
- [Operations and artifact lifecycle](docs/operations.md)
|
- [Operations and artifact lifecycle](docs/operations.md)
|
||||||
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
||||||
- [Output files and report formats](docs/integrations/output-files.md)
|
- [Output files and report formats](docs/integrations/output-files.md)
|
||||||
|
|||||||
374
cmd/audita/correction_repair_integration_test.go
Normal file
374
cmd/audita/correction_repair_integration_test.go
Normal file
@@ -0,0 +1,374 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
)
|
||||||
|
|
||||||
|
type repairWireMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
type repairWireRequest struct {
|
||||||
|
Messages []repairWireMessage `json:"messages"`
|
||||||
|
ResponseFormat struct {
|
||||||
|
JSONSchema struct {
|
||||||
|
Schema struct {
|
||||||
|
Properties map[string]any `json:"properties"`
|
||||||
|
} `json:"schema"`
|
||||||
|
} `json:"json_schema"`
|
||||||
|
} `json:"response_format"`
|
||||||
|
}
|
||||||
|
type correctionProvider struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
requests []repairWireRequest
|
||||||
|
attempts map[int]int
|
||||||
|
entered chan struct{}
|
||||||
|
wait bool
|
||||||
|
invalidFirstRevision bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req repairWireRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
http.Error(w, "bad request", 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, repair := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]
|
||||||
|
_, validation := req.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
||||||
|
p.mu.Lock()
|
||||||
|
p.requests = append(p.requests, req)
|
||||||
|
p.mu.Unlock()
|
||||||
|
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello","confidence":0.99},{"id":2,"original_text":"second","corrected_text":"Second","confidence":0.1},{"id":3,"original_text":"third","corrected_text":"Third","confidence":0.1}]}`
|
||||||
|
if validation {
|
||||||
|
var indices []int
|
||||||
|
for _, message := range req.Messages {
|
||||||
|
if pos := strings.LastIndex(message.Content, "Corrections to validate:"); pos >= 0 {
|
||||||
|
var items []struct {
|
||||||
|
CorrectionIndex int `json:"correction_index"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[pos+len("Corrections to validate:"):])), &items); err != nil {
|
||||||
|
http.Error(w, "bad validation", 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, item := range items {
|
||||||
|
indices = append(indices, item.CorrectionIndex)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var decisions []map[string]any
|
||||||
|
for _, index := range indices {
|
||||||
|
decisions = append(decisions, map[string]any{"correction_index": index, "approved": true, "confidence": 0.99, "reason": "controlled"})
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||||
|
content = string(raw)
|
||||||
|
}
|
||||||
|
if repair {
|
||||||
|
if p.wait {
|
||||||
|
select {
|
||||||
|
case p.entered <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
<-r.Context().Done()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ref := 2
|
||||||
|
if strings.Contains(req.Messages[len(req.Messages)-1].Content, `"proposal_ref":3`) {
|
||||||
|
ref = 3
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
p.attempts[ref]++
|
||||||
|
attempt := p.attempts[ref]
|
||||||
|
invalid := p.invalidFirstRevision && attempt == 1
|
||||||
|
p.mu.Unlock()
|
||||||
|
if ref == 3 {
|
||||||
|
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
confidence := 0.1
|
||||||
|
if attempt > 1 {
|
||||||
|
confidence = 0.99
|
||||||
|
}
|
||||||
|
id := 2
|
||||||
|
if invalid {
|
||||||
|
id, confidence = 99, 0.99
|
||||||
|
}
|
||||||
|
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second","confidence":%v}}`, id, confidence)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(content)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprintf(w, `{"model":"fixture","choices":[{"message":{"content":%s}}]}`, raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessReportsDefectOnOriginatingRepair(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
budget int
|
||||||
|
outcome string
|
||||||
|
recoveries int
|
||||||
|
}{{1, "exhausted", 1}, {2, "applied", 2}} {
|
||||||
|
t.Run(tc.outcome, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, tc.budget, false, false)
|
||||||
|
provider.mu.Lock()
|
||||||
|
provider.invalidFirstRevision = true
|
||||||
|
provider.mu.Unlock()
|
||||||
|
if code, _, stderr := f.run(t, true); code != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
doc := f.recoveries(t)
|
||||||
|
if *doc.Summary.ProposalResponses != 1 || *doc.Summary.ProposalResponsesWithFiltering != 0 || len(doc.Recoveries) != tc.recoveries {
|
||||||
|
t.Fatalf("unexpected recoveries: %+v", doc)
|
||||||
|
}
|
||||||
|
first := doc.Recoveries[0]
|
||||||
|
want := []reporting.FilteredProposal{{ProposalIndex: 1, Reason: "invalid_target_segment_id", Count: 1}}
|
||||||
|
if first.Operation != "correction_repair" || !reflect.DeepEqual(first.FilteredProposals, want) || !reflect.DeepEqual(first.Reasons, []string{"proposals_filtered"}) {
|
||||||
|
t.Fatalf("first repair recovery: %+v", first)
|
||||||
|
}
|
||||||
|
if tc.recoveries == 2 && !reflect.DeepEqual(doc.Recoveries[1].Reasons, []string{"correction_revised"}) {
|
||||||
|
t.Fatalf("successful repair recovery: %+v", doc.Recoveries[1])
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil || !strings.Contains(string(report), fmt.Sprintf("| 1 | 1 | 1 | %d | %s |", tc.budget, tc.outcome)) {
|
||||||
|
t.Fatalf("repair outcome: %s %v", report, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func setupCorrectionFixture(t *testing.T, attempts int, replacement, wait bool) (processFixture, *correctionProvider) {
|
||||||
|
t.Helper()
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
f.setSegments(t, "hello world", "second text", "third text")
|
||||||
|
provider := &correctionProvider{attempts: map[int]int{}, entered: make(chan struct{}, 1), wait: wait}
|
||||||
|
server := httptest.NewServer(provider)
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
resources := fmt.Sprintf("profiles:\n - id: proposal\n endpoint: %s/v1\n model: fixture\n - id: validation\n endpoint: %s/v1\n model: fixture\n", server.URL, server.URL)
|
||||||
|
if replacement {
|
||||||
|
resources += "collections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(f.dir, "repair.txt"), []byte("OVERRIDE {{ .RepairRequestJSON }}"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
text := strings.Replace(string(data), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", attempts), 1)
|
||||||
|
text += "chunking:\n target_sections: 1\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
|
if err := os.WriteFile(f.config, []byte(text), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return f, provider
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessCorrectionRepairConversation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
debug, replacement bool
|
||||||
|
}{{"default prompt without debug", false, false}, {"replacement with debug", true, true}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 2, tc.replacement, false)
|
||||||
|
args := []string{"--verbose"}
|
||||||
|
if tc.debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 || stdout != "" {
|
||||||
|
t.Fatalf("code=%d stdout=%s stderr=%s", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
output := f.readTranscript(t, f.output)
|
||||||
|
if len(output.Segments) != 3 || output.Segments[0].Text != "Hello world" || output.Segments[1].Text != "Second text" || output.Segments[2].Text != "third text" {
|
||||||
|
t.Fatalf("transcript: %+v", output)
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{"## Correction repair", "| 1 | 1 | 1 | 2 | applied |", "| 1 | 1 | 2 | 1 | unavailable |"} {
|
||||||
|
if !strings.Contains(string(report), fragment) {
|
||||||
|
t.Errorf("missing %q in %s", fragment, report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
doc := f.errors(t)
|
||||||
|
recoveries := f.recoveries(t)
|
||||||
|
if doc.Summary.LLMOperations == nil || *doc.Summary.LLMOperations != 10 || *doc.Summary.CorrectionsApplied != 2 || *doc.Summary.Recoveries != 1 || !reflect.DeepEqual(doc.Summary, recoveries.Summary) || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
|
t.Fatalf("counts: errors=%+v recoveries=%+v stderr=%s", doc.Summary, recoveries.Summary, stderr)
|
||||||
|
}
|
||||||
|
if len(doc.Errors) != 1 || len(doc.Errors[0].Failures) != 1 || doc.Errors[0].Failures[0].Operation != "correction_repair" || doc.Errors[0].Failures[0].Reason != "provider_failed" {
|
||||||
|
t.Fatalf("errors: %+v", doc)
|
||||||
|
}
|
||||||
|
provider.mu.Lock()
|
||||||
|
requests := append([]repairWireRequest(nil), provider.requests...)
|
||||||
|
provider.mu.Unlock()
|
||||||
|
var base []repairWireMessage
|
||||||
|
var branches [][]repairWireMessage
|
||||||
|
for _, req := range requests {
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["corrections"]; ok {
|
||||||
|
base = req.Messages
|
||||||
|
}
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
|
||||||
|
branches = append(branches, req.Messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(base) != 2 || len(branches) != 3 || len(branches[0]) != 4 || len(branches[1]) != 6 || len(branches[2]) != 4 {
|
||||||
|
t.Fatalf("request lengths base=%d branches=%v", len(base), branches)
|
||||||
|
}
|
||||||
|
for _, messages := range branches {
|
||||||
|
for i := range base {
|
||||||
|
if messages[i] != base[i] {
|
||||||
|
t.Fatalf("changed prefix: %+v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if messages[2].Role != "assistant" || !strings.Contains(messages[2].Content, `"corrections"`) {
|
||||||
|
t.Fatalf("missing initial response: %+v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if branches[1][3] != branches[0][3] || branches[1][4].Role != "assistant" || strings.Contains(branches[2][3].Content, `"proposal_ref":2`) {
|
||||||
|
t.Fatalf("branch history: %+v", branches)
|
||||||
|
}
|
||||||
|
if tc.replacement && !strings.Contains(branches[0][3].Content, "OVERRIDE") {
|
||||||
|
t.Fatal("replacement not used")
|
||||||
|
}
|
||||||
|
if tc.debug {
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRepairDisabledAndOutputSelections(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
reporting, debug bool
|
||||||
|
}{{"neither", false, false}, {"debug only", false, true}, {"report only", true, false}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 0, false, false)
|
||||||
|
if !tc.reporting {
|
||||||
|
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(file, "reporting:\n enabled: false")
|
||||||
|
file.Close()
|
||||||
|
}
|
||||||
|
var args []string
|
||||||
|
if tc.debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, _, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
provider.mu.Lock()
|
||||||
|
requests := append([]repairWireRequest(nil), provider.requests...)
|
||||||
|
provider.mu.Unlock()
|
||||||
|
for _, req := range requests {
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
|
||||||
|
t.Fatal("repair request with zero budget")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := os.Stat(filepath.Join(f.dir, "report.md"))
|
||||||
|
if tc.reporting != (err == nil) {
|
||||||
|
t.Fatalf("report selection: %v", err)
|
||||||
|
}
|
||||||
|
if tc.debug {
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) == 0 {
|
||||||
|
t.Fatal("debug ledger missing")
|
||||||
|
}
|
||||||
|
} else if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected debug dir: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessCancellationDuringCorrectionRepair(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 2, false, true)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output, "--debug"}
|
||||||
|
command := exec.CommandContext(ctx, os.Args[0], args...)
|
||||||
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
command.Stderr = &stderr
|
||||||
|
if err := command.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-provider.entered:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("repair not reached: %v", ctx.Err())
|
||||||
|
}
|
||||||
|
if err := command.Process.Signal(os.Interrupt); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := command.Wait()
|
||||||
|
if ctx.Err() != nil || err == nil {
|
||||||
|
t.Fatalf("cancellation: %v ctx=%v stderr=%s", err, ctx.Err(), stderr.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("transcript published: %v", err)
|
||||||
|
}
|
||||||
|
doc := f.errors(t)
|
||||||
|
if doc.Status != reporting.StatusFailed || len(doc.Errors) == 0 {
|
||||||
|
t.Fatalf("failed report: %+v", doc)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil || !strings.Contains(string(data), "| 1 | 1 | 1 | 1 | interrupted |") {
|
||||||
|
t.Fatalf("partial report %s %v", data, err)
|
||||||
|
}
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) == 0 {
|
||||||
|
t.Fatal("partial ledger missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRepairWithoutOrdinaryReporting(t *testing.T) {
|
||||||
|
for _, debug := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) {
|
||||||
|
f, _ := setupCorrectionFixture(t, 2, false, false)
|
||||||
|
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(file, "reporting:\n enabled: false")
|
||||||
|
file.Close()
|
||||||
|
args := []string{"--verbose"}
|
||||||
|
if debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, _, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(f.dir, "report.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected report: %v", err)
|
||||||
|
}
|
||||||
|
if debug {
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -122,6 +122,11 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||||
|
if p.mode == "invalid-credential-echo" {
|
||||||
|
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||||
|
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
|
||||||
|
content = string(raw)
|
||||||
|
}
|
||||||
if p.mode == "repair-filter-later-failure" && !validation {
|
if p.mode == "repair-filter-later-failure" && !validation {
|
||||||
if proposalCall == 1 {
|
if proposalCall == 1 {
|
||||||
content = `{"wrong":true}`
|
content = `{"wrong":true}`
|
||||||
@@ -231,17 +236,15 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
|
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
|
||||||
|
if mode == "invalid-credential-echo" {
|
||||||
|
p.secret = " synthetic-echo-credential\t "
|
||||||
|
}
|
||||||
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, id := range []string{"proposal", "validation"} {
|
|
||||||
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
configPath := filepath.Join(dir, "config.yml")
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
repairs := 0
|
repairs := 0
|
||||||
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
||||||
@@ -250,7 +253,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
repairs = 1
|
repairs = 1
|
||||||
}
|
}
|
||||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
var stageList []string
|
||||||
|
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
|
||||||
|
if name = strings.TrimSpace(name); name != "" {
|
||||||
|
stageList = append(stageList, " - module: "+name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pipeline := " stages: []\n"
|
||||||
|
if len(stageList) != 0 {
|
||||||
|
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
|
||||||
|
}
|
||||||
|
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n correction_repair_attempts: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
|
||||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
}
|
}
|
||||||
@@ -271,7 +284,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
|
|
||||||
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
|
||||||
if includeOutput {
|
if includeOutput {
|
||||||
args = append(args, "--output", f.output)
|
args = append(args, "--output", f.output)
|
||||||
}
|
}
|
||||||
@@ -775,7 +788,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
|
|||||||
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true)
|
||||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
|
if code != 0 || stdout != "" || stderr != "" {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
@@ -807,7 +820,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
||||||
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
|
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
@@ -830,7 +843,7 @@ func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testin
|
|||||||
}
|
}
|
||||||
f.output = filepath.Join(transcriptDir, "corrected.json")
|
f.output = filepath.Join(transcriptDir, "corrected.json")
|
||||||
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
|
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
|
||||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||||
@@ -858,43 +871,35 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
for _, selection := range []struct{ reporting, debug bool }{
|
for _, selection := range []struct{ reporting, debug bool }{
|
||||||
{false, false}, {false, true}, {true, false}, {true, true},
|
{false, false}, {false, true}, {true, false}, {true, true},
|
||||||
} {
|
} {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
||||||
if selection.reporting && selection.debug {
|
|
||||||
original, err := os.ReadFile(f.config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
|
|
||||||
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
|
|
||||||
updated := strings.Replace(string(original), old, inherited, 1)
|
|
||||||
if updated == string(original) {
|
|
||||||
t.Fatal("process fixture backend configuration was not found")
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reportDir := filepath.Join(f.dir, "missing")
|
reportDir := filepath.Join(f.dir, "missing")
|
||||||
extra := []string{"--reporting=false", "--report-dir", reportDir}
|
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
|
||||||
if selection.reporting {
|
if selection.reporting {
|
||||||
reportDir = filepath.Join(f.dir, "selected-reports")
|
reportDir = filepath.Join(f.dir, "selected-reports")
|
||||||
if err := os.Mkdir(reportDir, 0700); err != nil {
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
extra = []string{"--reporting=true", "--report-dir", reportDir}
|
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
|
||||||
}
|
}
|
||||||
if selection.debug {
|
if selection.debug {
|
||||||
extra = append(extra, "--debug")
|
extra = append(extra, "--debug")
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, true, extra...)
|
code, stdout, stderr := f.run(t, true, extra...)
|
||||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
summary := strings.SplitN(stderr, "\n", 2)[0]
|
summary := ""
|
||||||
|
for _, line := range strings.Split(stderr, "\n") {
|
||||||
|
if strings.HasPrefix(line, "Completed:") {
|
||||||
|
summary = line
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
if expectedOutput == "" {
|
if expectedOutput == "" {
|
||||||
expectedOutput, expectedSummary = string(output), summary
|
expectedOutput, expectedSummary = string(output), summary
|
||||||
} else if string(output) != expectedOutput || summary != expectedSummary {
|
} else if string(output) != expectedOutput || summary != expectedSummary {
|
||||||
@@ -916,31 +921,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if selection.reporting {
|
|
||||||
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
|
|
||||||
t.Fatalf("effective inherited settings: %s", effective)
|
|
||||||
}
|
|
||||||
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
|
|
||||||
if !strings.Contains(string(origins), want) {
|
|
||||||
t.Fatalf("configuration origins missing %q: %s", want, origins)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Contains(string(origins), f.provider.secret) {
|
|
||||||
t.Fatal("configuration origins exposed the resolved inherited credential")
|
|
||||||
}
|
|
||||||
code, explain, stderr := f.runConfigView(t, "explain")
|
|
||||||
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
|
|
||||||
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -954,7 +935,7 @@ func boolInt(value bool) int {
|
|||||||
|
|
||||||
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
||||||
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -971,9 +952,41 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "invalid-credential-echo", "[grammar]")
|
||||||
|
effective := strings.TrimSpace(f.provider.secret)
|
||||||
|
code, stdout, stderr := f.run(t, true, "--debug")
|
||||||
|
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
|
||||||
|
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
|
||||||
|
}
|
||||||
|
if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) {
|
||||||
|
t.Fatal("credential escaped redaction in command output")
|
||||||
|
}
|
||||||
|
responseCaptured := false
|
||||||
|
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
|
||||||
|
if err != nil || entry.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if bytes.Contains(data, []byte(effective)) {
|
||||||
|
t.Errorf("credential escaped redaction in %s", entry.Name())
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) {
|
||||||
|
responseCaptured = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || !responseCaptured {
|
||||||
|
t.Fatalf("missing redacted provider echo: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
||||||
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -989,7 +1002,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
||||||
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
|
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
|
||||||
}
|
}
|
||||||
@@ -1019,7 +1032,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
|
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
|
||||||
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
|
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
|
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
@@ -1037,7 +1050,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
|
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
|
||||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer stop()
|
defer stop()
|
||||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
|
||||||
command := exec.CommandContext(ctx, os.Args[0], args...)
|
command := exec.CommandContext(ctx, os.Args[0], args...)
|
||||||
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -1067,7 +1080,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
|
|
||||||
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
||||||
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
@@ -1081,55 +1094,41 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, _, _ := f.run(t, true)
|
code, _, _ := f.run(t, true)
|
||||||
if code != 1 || f.provider.callCount() != 0 {
|
if code != 2 || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||||
}
|
}
|
||||||
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
|
||||||
t.Fatalf("failed report: %+v", document)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
t.Fatalf("failed transcript published: %v", err)
|
t.Fatalf("failed transcript published: %v", err)
|
||||||
}
|
}
|
||||||
var recoveries reporting.RecoveriesDocument
|
|
||||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if recoveries.Status != reporting.StatusFailed {
|
|
||||||
t.Fatalf("recoveries status: %+v", recoveries)
|
|
||||||
}
|
|
||||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
|
||||||
t.Fatalf("Markdown status: %s %v", raw, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
|
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
original, err := os.ReadFile(f.config)
|
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
|
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
|
||||||
if updated == string(original) {
|
if updated == string(original) {
|
||||||
t.Fatal("process fixture backend configuration was not found")
|
t.Fatal("process fixture backend configuration was not found")
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true)
|
||||||
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
|
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
|
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -1139,25 +1138,13 @@ func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, _, _ := f.run(t, true, "--report-dir", reportDir)
|
code, _, _ := f.run(t, true, "--report-dir", reportDir)
|
||||||
if code != 1 || f.provider.callCount() != 0 {
|
if code != 2 || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||||
}
|
}
|
||||||
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
if err != nil {
|
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
|
||||||
t.Fatal(err)
|
t.Fatalf("report created before preflight: %s %v", name, err)
|
||||||
}
|
}
|
||||||
var document reporting.ErrorsDocument
|
|
||||||
if err := yaml.Unmarshal(raw, &document); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
|
||||||
t.Fatalf("failed report: %+v", document)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
|
||||||
t.Fatalf("Markdown status: %s %v", raw, err)
|
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("failed report appeared in default directory: %v", err)
|
t.Fatalf("failed report appeared in default directory: %v", err)
|
||||||
@@ -1182,7 +1169,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
|||||||
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
||||||
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
|
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
|
||||||
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(f.output); err != nil {
|
if _, err := os.Stat(f.output); err != nil {
|
||||||
@@ -1197,7 +1184,7 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
|
|||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||||
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
|
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
|
||||||
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1256,12 +1243,12 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
|||||||
selected, unused := "selected-path-secret", "unused-path-secret"
|
selected, unused := "selected-path-secret", "unused-path-secret"
|
||||||
f.provider.secret = selected
|
f.provider.secret = selected
|
||||||
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
|
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
|
||||||
cfg, err := os.ReadFile(f.config)
|
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
|
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
|
||||||
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
reportDir := filepath.Join(f.dir, unused)
|
reportDir := filepath.Join(f.dir, unused)
|
||||||
@@ -1276,20 +1263,20 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
|
||||||
if (!fail && code != 0) || (fail && code != 1) {
|
if (!fail && code != 0) || (fail && code != 2) {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
|
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
|
||||||
t.Fatalf("credential in stderr: %q", stderr)
|
t.Fatalf("credential in stderr: %q", stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr, "[REDACTED]") {
|
if !fail && !strings.Contains(stderr, "[REDACTED]") {
|
||||||
t.Fatalf("missing redacted announcement: %q", stderr)
|
t.Fatalf("missing redacted announcement: %q", stderr)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
|
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
dirs, err := os.ReadDir(debugParent)
|
dirs, err := os.ReadDir(debugParent)
|
||||||
if err != nil || len(dirs) != 1 {
|
if !fail && (err != nil || len(dirs) != 1) {
|
||||||
t.Fatalf("debug location: %v %v", dirs, err)
|
t.Fatalf("debug location: %v %v", dirs, err)
|
||||||
}
|
}
|
||||||
if !fail {
|
if !fail {
|
||||||
@@ -1303,7 +1290,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, _, baselineSummary := f.run(t, true)
|
code, _, baselineSummary := f.run(t, true, "--verbose")
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatal(baselineSummary)
|
t.Fatal(baselineSummary)
|
||||||
}
|
}
|
||||||
@@ -1318,16 +1305,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
|||||||
t.Errorf("debug directory: %v %v", dirs, err)
|
t.Errorf("debug directory: %v %v", dirs, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.provider.mu.Unlock()
|
f.provider.mu.Unlock()
|
||||||
code, _, stderr := f.run(t, true, "--debug")
|
code, _, stderr := f.run(t, true, "--debug")
|
||||||
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
|
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
|
||||||
t.Fatalf("changed outcome: %d %q", code, stderr)
|
t.Fatalf("changed outcome: %d %q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
|
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
|
||||||
t.Fatalf("missing debug failures: %q", stderr)
|
t.Fatalf("missing debug failures: %q", stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
@@ -1360,7 +1347,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if invalid {
|
if invalid {
|
||||||
if code != 1 || !bytes.Equal(before, after) {
|
if code != 2 || !bytes.Equal(before, after) {
|
||||||
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
||||||
}
|
}
|
||||||
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
||||||
@@ -1374,15 +1361,18 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
f := newProcessFixture(t, "success", "[]")
|
f := newProcessFixture(t, "success", "[]")
|
||||||
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
||||||
// still an input and must not be overwritten during report initialization.
|
// still an input and must not be overwritten during report initialization.
|
||||||
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
|
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
|
||||||
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
|
||||||
|
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
before, err := os.ReadFile(loaded)
|
before, err := os.ReadFile(loaded)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
|
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
|
||||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
||||||
@@ -1396,12 +1386,12 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
parentDir := filepath.Join(f.dir, "profiles", "catalog")
|
parentDir := filepath.Join(f.dir, "prompts", "catalog")
|
||||||
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
|
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
|
||||||
for _, directory := range []string{parentDir, childDir} {
|
for _, directory := range []string{parentDir, childDir} {
|
||||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -1411,8 +1401,8 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
child := filepath.Join(childDir, "override.yaml")
|
child := filepath.Join(childDir, "override.yaml")
|
||||||
resource := filepath.Join(parentDir, "report.md")
|
resource := filepath.Join(parentDir, "report.md")
|
||||||
for path, content := range map[string]string{
|
for path, content := range map[string]string{
|
||||||
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
|
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
|
||||||
child: "prompts:\n literal/~child:\n extends: source\n",
|
child: "prompts:\n - id: child\n extends: {id: source}\n",
|
||||||
resource: "Captured operator message.\n",
|
resource: "Captured operator message.\n",
|
||||||
} {
|
} {
|
||||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||||
@@ -1429,7 +1419,7 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
|
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1439,7 +1429,7 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
42
docs/adr/0002-glossary-configuration-and-preflight.md
Normal file
42
docs/adr/0002-glossary-configuration-and-preflight.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# ADR 0002: Glossary configuration and processing readiness
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
**Date:** 2026-11-10
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Operators need to reuse glossary collections across campaigns and select different
|
||||||
|
context and protected terms for each ordered correction. Configuration inspection
|
||||||
|
must remain useful without a transcript, model credentials, or a populated glossary.
|
||||||
|
A processing run must reject an unusable later correction before publishing outputs
|
||||||
|
or spending resources on earlier corrections.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
ConfigKit composes typed, named glossary collections with the application
|
||||||
|
configuration. Each pipeline entry selects context and protection independently;
|
||||||
|
Audita derives consumer views without changing the resolved configuration. For a
|
||||||
|
glossary correction, active recognized spellings are exempt from that entry's
|
||||||
|
protection view. Other protected spellings and validators remain effective.
|
||||||
|
|
||||||
|
After configuration resolution, Audita preflight checks the whole pipeline and
|
||||||
|
coordinates output preparation. It consumes the typed configuration and acquired
|
||||||
|
inputs, without reloading configuration or calling a provider. The output session
|
||||||
|
owns preparation, cleanup, and publication. See the [configuration](../config.md)
|
||||||
|
and [operations](../operations.md) contracts for current behavior.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
A separately parsed glossary per invocation would duplicate composition and make
|
||||||
|
per-entry selection and provenance harder to express. Requiring nonempty
|
||||||
|
collections in configuration validation would prevent inspection and reuse of
|
||||||
|
valid empty definitions. Preparing outputs or checking each entry only when it
|
||||||
|
runs could leave artifacts or incur model work before finding a later error.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Configuration and runtime views have distinct responsibilities. Inspection can
|
||||||
|
show configured selections and origins without presenting derived vocabulary as
|
||||||
|
configuration. Preflight and the output session need a clear handoff, and the
|
||||||
|
processing pipeline must carry each entry's own effective settings and identity.
|
||||||
59
docs/adr/0003-module-focus-and-editorial-acceptance.md
Normal file
59
docs/adr/0003-module-focus-and-editorial-acceptance.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# ADR 0003: Module focus and editorial acceptance
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
**Date:** 2026-11-11
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Audita gives each correction module a focused task. A prompt review suggested
|
||||||
|
making that task's scope an acceptance condition, because validators can accept
|
||||||
|
minor editorial changes that the proposing module was not asked to make.
|
||||||
|
|
||||||
|
The maintainer's practical testing established why this difference is deliberate.
|
||||||
|
Models persistently propose minor editorial cleanup when it is clearly warranted
|
||||||
|
in the supplied text. Instructions strict enough to suppress that cleanup also
|
||||||
|
suppress desirable edits. Enforcing exclusive module scopes would therefore
|
||||||
|
trade away useful corrections to achieve a separation that does not serve the
|
||||||
|
application's editorial goals.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Module scope guides the corrections Audita asks a model to propose. It is not an
|
||||||
|
exhaustive boundary on acceptable edits. Minor editorial cleanup is permitted
|
||||||
|
across modules and their validators even when the proposal prompt does not
|
||||||
|
request it.
|
||||||
|
|
||||||
|
Validators must not reject an otherwise acceptable minor editorial change solely
|
||||||
|
because it falls outside the originating module's requested focus. Existing
|
||||||
|
checks for applicability, protected terms, meaning, and other domain concerns
|
||||||
|
continue to apply. This allowance does not guarantee that any particular edit
|
||||||
|
passes validation or application, nor does it authorize arbitrary rewriting.
|
||||||
|
|
||||||
|
Proposal, validation, and repair instructions may differ to serve their roles.
|
||||||
|
Their consistency should be assessed against this distinction, rather than by
|
||||||
|
requiring identical lists of allowed edits. Repair feedback should not turn
|
||||||
|
module focus into an exclusive acceptance rule.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
**Strict module-aware acceptance**, whether implemented in shared validators,
|
||||||
|
separate review definitions, or an additional scope check, would make stage
|
||||||
|
boundaries easier to describe. It is rejected because the maintainer's testing
|
||||||
|
shows that suppressing incidental cleanup also suppresses desirable work.
|
||||||
|
|
||||||
|
**Request every kind of cleanup in every module** would align proposal requests
|
||||||
|
with the broader acceptance policy, but would dilute the focused tasks. Permitting
|
||||||
|
incidental cleanup does not require asking each stage to perform every task.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
An edit accepted outside a module's requested focus is not, by itself, a design
|
||||||
|
defect. Prompt maintenance and evaluation should preserve useful corrections
|
||||||
|
while checking substantive editorial quality. They should not introduce strict
|
||||||
|
scope enforcement as an apparent consistency fix.
|
||||||
|
|
||||||
|
The [architecture policy](../policy/architecture.md#correction-and-concurrency-invariants)
|
||||||
|
records the boundary. Reconsidering this accepted decision requires new evidence
|
||||||
|
and explicit supersession under the ADR lifecycle, rather than rediscovering the
|
||||||
|
same prompt/validator difference during each review.
|
||||||
@@ -4,6 +4,10 @@
|
|||||||
|
|
||||||
- [ADR 0001: Adopt ReleaseKit for release management](0001-releasekit.md) —
|
- [ADR 0001: Adopt ReleaseKit for release management](0001-releasekit.md) —
|
||||||
accepted 2026-10-07.
|
accepted 2026-10-07.
|
||||||
|
- [ADR 0002: Glossary configuration and processing readiness](0002-glossary-configuration-and-preflight.md) —
|
||||||
|
accepted 2026-11-10.
|
||||||
|
- [ADR 0003: Module focus and editorial acceptance](0003-module-focus-and-editorial-acceptance.md) —
|
||||||
|
accepted 2026-11-11.
|
||||||
|
|
||||||
Use the
|
Use the
|
||||||
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
||||||
|
|||||||
139
docs/cli.md
139
docs/cli.md
@@ -1,116 +1,45 @@
|
|||||||
# Audita CLI Reference
|
# Audita CLI Reference
|
||||||
|
|
||||||
The shared version-2 configuration parser and offline profile inspection serve
|
Audita uses CommandKit for routing and the ConfigKit and PromptKit integrations
|
||||||
`process`, `config validate`, and `config print-effective`.
|
for their standard inspection and request commands. Run `audita --help` or
|
||||||
The [configuration reference](config.md) owns fields, defaults, and validation;
|
command-specific `--help` for the installed command inventory. The adopted
|
||||||
the [migration guide](migrations/v2.md) maps retired inputs.
|
[CommandKit consumer guide](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/consumers/guide.md)
|
||||||
|
describes the standard command grammar, inspection, comparison, preview,
|
||||||
|
validation, smoke tests, and their provider-specific caveats. Help and version
|
||||||
|
do not load application or model configuration.
|
||||||
|
|
||||||
## Commands and help
|
## Processing
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
audita process <transcript.json> --glossary <glossary.yml> [flags]
|
audita process [flags] transcript.json
|
||||||
audita config validate --config <config.yml> [flags]
|
|
||||||
audita config print-effective [flags]
|
|
||||||
audita version
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Root invocation, `--help`, `-h`, and command help succeed without loading a config
|
`--config` selects a main configuration and `--overlay` selects its overlay.
|
||||||
file or resolving secrets. Unknown commands and flags return exit code `2`.
|
`--output` writes the transcript to a file; omission selects stdout. The shared
|
||||||
`process` accepts exactly one transcript path, either before the flags or after
|
`--debug` flag enables retained debug artifacts, and `--quiet` suppresses optional
|
||||||
them. With flags first, put the transcript last. Both config commands reject
|
messages. Quiet processing requires an output file or `/dev/null`. See
|
||||||
positional arguments and process-only flags.
|
[operations](operations.md) for output and report lifecycle.
|
||||||
|
|
||||||
All three commands accept `--config <path>` and `--overlay <name>`. `config validate`
|
Explicit process flags override root settings only. `--glossaries` and
|
||||||
requires an explicit nonempty `--config`; the other commands select the explicit
|
`--protected-terms` each accept one JSON array of collection names or `null`;
|
||||||
file, then `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`.
|
`[]` clears the selection. Set pipeline order and per-entry overrides in YAML.
|
||||||
Missing system candidates are skipped; inaccessible selected files are errors.
|
Other supplied flags cover profiles, output schema, description, reporting,
|
||||||
No file means a configuration error. `AUDITA_CONFIG` is ignored. The selected
|
chunking, normalization, confidence thresholds, validation budget, and debug
|
||||||
overlay is loaded beside the main file, not used to locate profile files.
|
work directory. `audita process --help` lists the available flags. An omitted flag
|
||||||
|
does not contribute an override. [Configuration](config.md) defines their values
|
||||||
|
and inheritance.
|
||||||
|
|
||||||
`audita version` prints `audita <version>` and a newline to stdout, with no
|
## Inspection and outcomes
|
||||||
application configuration or provider setup. It returns `0` on success with
|
|
||||||
empty stderr. `help version`, `version -h`, and `version --help` print command
|
|
||||||
usage to stdout and return `0`. Help follows standard flag parsing and can
|
|
||||||
short-circuit trailing arguments. Other invalid invocations return `2` with a
|
|
||||||
usage error on stderr. The command uses ReleaseKit's public
|
|
||||||
`buildinfo.Version()` accessor. It reports an injected release identifier when
|
|
||||||
present, otherwise the executable's main-module version when available and not
|
|
||||||
`(devel)`, and `dev` when neither source provides a release identifier.
|
|
||||||
|
|
||||||
## Setting overrides
|
The application configuration scope uses ConfigKit's native effective values,
|
||||||
|
source evidence, explanation, comparison, and redaction. PromptKit supplies its
|
||||||
|
own configuration and resource scopes and request preparation, validation, and
|
||||||
|
explicit smoke diagnostics. Inspection does not require a processing-ready
|
||||||
|
pipeline or contact a provider. Smoke tests can contact providers and are run
|
||||||
|
only on explicit request.
|
||||||
|
|
||||||
Only flags actually supplied override file or overlay settings. `--modules ''`
|
Process stdout contains transcript JSON when no output file is selected. Ordinary
|
||||||
clears the module pipeline, and an explicit numeric zero reaches configuration
|
reports may record recoverable model failures even when processing completes.
|
||||||
validation. The shared setting flags are:
|
Exit status `0` means success, `1` runtime or output failure, `2` invalid usage
|
||||||
|
or input, and native comparison may return `3` for differences. See the
|
||||||
| Setting | Flags |
|
[output contract](integrations/output-files.md) and [subprocess contract](integrations/subprocess.md).
|
||||||
| --- | --- |
|
|
||||||
| Profile selection | `--proposal-profile`, `--validation-profile` |
|
|
||||||
| Pipeline and output | `--modules`, `--output-schema` |
|
|
||||||
| Validation budget | `--validation-max-prompt-tokens` |
|
|
||||||
| Chunking | `--target-sections`, `--max-section-tokens`, `--min-section-tokens` |
|
|
||||||
| Normalization (numeric seconds) | `--normalize-max-segment-gap`, `--normalize-ellipsis-gap`, `--normalize-max-segment-duration`, `--normalize-max-segment-tokens` |
|
|
||||||
| Confidence thresholds | `--glossary-confidence-threshold`, `--homophones-confidence-threshold`, `--spoken-word-confidence-threshold`, `--grammar-confidence-threshold` |
|
|
||||||
| Context | `--transcript-description` |
|
|
||||||
| Reporting | `--reporting[=true\|false]`, `--report-dir DIR` |
|
|
||||||
| Debug | `--debug[=true\|false]`, `--work-dir DIR` |
|
|
||||||
|
|
||||||
`process` alone accepts `--glossary` (required) and `--output`.
|
|
||||||
An omitted or empty output value selects stdout. Nonempty output filenames are
|
|
||||||
used literally, including whitespace.
|
|
||||||
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
|
|
||||||
reports; an explicit false overrides a true file or overlay setting. For stdout
|
|
||||||
output, reports use the effective `reporting.stdout_dir`, which defaults to the
|
|
||||||
invocation working directory. For file output, they go beside the transcript
|
|
||||||
unless `--report-dir` is explicitly supplied. The flag selects the report
|
|
||||||
directory for either transcript destination; a config-file or overlay value for
|
|
||||||
`reporting.stdout_dir` alone affects stdout output only. A selected report
|
|
||||||
directory must be nonblank when reporting is enabled. With reporting disabled,
|
|
||||||
the report destination is inactive and is not probed. `--debug` separately
|
|
||||||
enables retained detailed artifacts under `--work-dir`; selecting `--work-dir`
|
|
||||||
alone does not enable capture. Configuration inspection does not create or
|
|
||||||
probe these output directories.
|
|
||||||
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
|
|
||||||
removed. Models and generation controls belong to PromptKit profiles, backend
|
|
||||||
credentials and capacity to configuration, and retry/deadline settings to the
|
|
||||||
shared LLM section. Legacy configuration environment overrides are ignored.
|
|
||||||
|
|
||||||
## Streams and exit status
|
|
||||||
|
|
||||||
`config validate` loads settings and both selected PromptKit profiles offline,
|
|
||||||
then prints `config is valid` on success. `config print-effective` accepts
|
|
||||||
`--view settings|profiles|explain` and defaults to `settings`. Settings writes
|
|
||||||
ConfigKit's redacted effective-settings YAML. Profiles writes an Audita-owned
|
|
||||||
YAML summary of selected ID, backend, model, sampling controls, per-attempt
|
|
||||||
timeout, service tier, and reasoning effort for each role. It omits credentials,
|
|
||||||
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
|
|
||||||
sequence of field paths, source origins, and recorded derivations without
|
|
||||||
resolved setting values. An inherited field may also include an `inheritance`
|
|
||||||
sequence; each step identifies `from`, `to`, and `reference` paths as JSON
|
|
||||||
pointers. These fields describe the effective inheritance chain without
|
|
||||||
including configuration values. A direct child override reports its own source
|
|
||||||
instead. Debug configuration origins use the same explanation shape. These views
|
|
||||||
share the same settings and profile validation path as `process`. Unknown views
|
|
||||||
are usage errors. The effective settings YAML is for inspection and is not
|
|
||||||
guaranteed reloadable.
|
|
||||||
|
|
||||||
For `process`, `--output` writes the corrected transcript to a file and leaves
|
|
||||||
stdout empty on success. Without it, stdout carries transcript JSON only on
|
|
||||||
success. Enabled reporting creates `errors.yml`, `recoveries.yml`, and
|
|
||||||
`report.md` in the selected directory. Successful stderr begins with
|
|
||||||
`Completed: N corrections applied; E work units with errors; R recoveries.`
|
|
||||||
Report and debug locations follow only when created. Model-operation failures
|
|
||||||
can coexist with a completed run and applied corrections; read the YAML reports
|
|
||||||
for details. Debug artifacts are created only with `--debug`. See the
|
|
||||||
[report contract](integrations/output-files.md).
|
|
||||||
|
|
||||||
Exit codes are `0` for success, `1` for runtime or output failure, and `2` for
|
|
||||||
usage or configuration input errors. Invalid settings and profiles return `2`;
|
|
||||||
broken embedded assets, caller cancellation, fatal processing, and required
|
|
||||||
output failures return `1`. A late report-write failure can follow a published
|
|
||||||
transcript; exit status and report status remain authoritative.
|
|
||||||
Configuration commands inspect profiles without contacting a provider.
|
|
||||||
|
|
||||||
See the [input files](integrations/transcript-glossary-files.md),
|
|
||||||
[subprocess contract](integrations/subprocess.md), and
|
|
||||||
[artifact lifecycle](operations.md) for their respective details.
|
|
||||||
|
|||||||
257
docs/config.md
257
docs/config.md
@@ -1,192 +1,105 @@
|
|||||||
# Audita Configuration
|
# Audita Configuration
|
||||||
|
|
||||||
This reference describes the version-2 configuration loader in `internal/core/config`.
|
Audita uses version 3 application configuration. See the [migration guide](migrations/v3.md)
|
||||||
Use the maintained [local](../examples/minimal-config.yml) and
|
and [examples](../examples/) for complete projects. Command syntax is owned by
|
||||||
[service](../examples/production-config.yml) examples with their separately
|
[the CLI reference](cli.md).
|
||||||
selected [local profiles](../examples/profiles/local/) and
|
|
||||||
[service profiles](../examples/profiles/service/), plus the optional
|
|
||||||
[testing overlay](../examples/overlays/testing.yml).
|
|
||||||
The [ConfigKit inheritance example](../examples/configkit-inheritance.yml)
|
|
||||||
shows a local child backend reusing and overriding sibling settings.
|
|
||||||
Choose real models for your endpoint; a testing profile makes no cost or quality
|
|
||||||
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
|
|
||||||
|
|
||||||
## Loading and precedence
|
## Loading and composition
|
||||||
|
|
||||||
The application selects one main YAML file: an explicit `--config`, then
|
An explicit `--config` selects one main YAML file. Without it, ConfigKit searches
|
||||||
`/usr/local/etc/audita/config.yml` if present, then `/etc/audita/config.yml`
|
the working directory, user configuration directory, and Unix system directories
|
||||||
if present. Missing system candidates are skipped; an explicit missing or empty
|
for `audita` configuration. If no file is found, built-in defaults apply. An
|
||||||
path, inaccessible selected file, or absence of any selected file is an error.
|
explicit invalid selection or failure of a selected file is an error; search does
|
||||||
`config validate` requires an explicit file. ConfigKit loads built-in defaults,
|
not continue after a selected file fails. An omitted version uses the v3 schema;
|
||||||
the selected file, an optional named overlay from `overlays/<name>.yml` beside the
|
a supplied version must be `3`.
|
||||||
main file, and only explicitly supplied setting overrides, in that order. Imports
|
|
||||||
and secret-directory bootstrap are disabled. Unknown fields and missing, null,
|
|
||||||
or non-integer `version: 2` are errors. Version-1 files require migration; no
|
|
||||||
compatibility loader or implicit environment configuration remains.
|
|
||||||
|
|
||||||
After overlay and explicit CLI override precedence is applied, ConfigKit resolves
|
Operator sources may use ConfigKit imports. The selected main file provides the
|
||||||
native sibling inheritance before typed decoding. Any input mapping may use
|
base for an optional `overlays/<name>.yml`, optional `glossary.yml`, and recursive
|
||||||
`extends: <sibling-name>`; the child keeps its key, inherits parent settings, and
|
`glossary/` YAML files. Without a main source, automatic companion files are not
|
||||||
overrides them with its own settings. Selected overlay and CLI values therefore
|
loaded from the working directory. ConfigKit composes sources, resolves native
|
||||||
participate in inheritance at their normal precedence. The generic syntax and
|
inheritance, applies explicit overrides, records derived defaults, and validates
|
||||||
resolution rules are defined by ConfigKit v0.7.1's
|
the result. Its [configuration reference](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md)
|
||||||
[sibling inheritance contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.1/docs/config.md#sibling-inheritance).
|
owns native composition and provenance semantics.
|
||||||
|
|
||||||
`extends` is reserved structural input in base files, imports, overlays, and
|
|
||||||
other application input mappings, including mapping values supplied by explicit
|
|
||||||
CLI setting overrides. ConfigKit consumes it before typed decoding. A previously
|
|
||||||
valid backend whose ID is literally `extends` cannot be represented under this
|
|
||||||
rule. Rename that backend to a non-conflicting ID and update every PromptKit
|
|
||||||
profile that selects it. This compatibility impact means the upgrade does not
|
|
||||||
promise that every previously valid configuration remains valid.
|
|
||||||
|
|
||||||
Inherited values count as explicit input, including zero, false, empty, and
|
|
||||||
null. Existing field rules still apply: for example, inherited backend
|
|
||||||
concurrency must be positive, while an inherited queue capacity of zero is
|
|
||||||
valid. Parent backends undergo the same decoding and application validation as
|
|
||||||
other backends, whether or not a selected profile uses them. A child
|
|
||||||
`credential: null` removes the child's inherited credential; it does not remove
|
|
||||||
the parent's credential reference, which remains subject to ordinary secret
|
|
||||||
resolution.
|
|
||||||
|
|
||||||
Validation runs after overlays, explicit overrides, and derived settings have
|
|
||||||
resolved. An override can therefore repair an invalid lower-precedence setting
|
|
||||||
when that setting can be decoded. ConfigKit retains field origins and derivations
|
|
||||||
for effective-configuration inspection. Its effective YAML redacts resolved
|
|
||||||
credentials.
|
|
||||||
|
|
||||||
### PromptKit profiles and operator prompts
|
### PromptKit profiles and operator prompts
|
||||||
|
|
||||||
`profiles_dir` selects a PromptKit configuration root, resolved against the
|
PromptKit loads its default catalog and Audita's embedded prompts, followed by
|
||||||
selected main file's directory unless it is absolute. The root may contain
|
optional `prompts.yml` and recursive `prompts/` YAML files beside the application
|
||||||
keyed `profiles`, `backends`, and `prompts` definitions; it is not limited to
|
main. Native imports, resource collections, inheritance, and replacement apply.
|
||||||
profile-only files. IDs come from literal YAML mapping keys, never filenames.
|
Profiles, backends, models, credentials, admission, and provider parameters belong
|
||||||
Multiple definitions may share a document, and a document may contribute to any
|
to PromptKit configuration; see its [format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/formats.md).
|
||||||
of the supported mappings. Directory discovery is recursive, but discovery
|
The engine captures its sources for an invocation. Edit files and run a new
|
||||||
order does not establish field precedence. Contributions within each source
|
invocation to pick up changes. Inspection does not require model credentials.
|
||||||
group must compose without conflicting values.
|
|
||||||
|
|
||||||
PromptKit assembles sources in this order: its selected catalogs and owned
|
## Pipeline and glossary selections
|
||||||
defaults, Audita's embedded prompt/schema inputs, operator documents under
|
|
||||||
`profiles_dir`, then the application backend mapping. The application mapping
|
|
||||||
owns each configured endpoint, concurrency limit, queue capacity, and credential
|
|
||||||
environment lookup. Other supported provider fields can be contributed by the
|
|
||||||
operator documents. Imports remain disabled. The current maintained examples
|
|
||||||
select separate roots: [local](../examples/profiles/local/) and
|
|
||||||
[service](../examples/profiles/service/), so definitions for one backend do not
|
|
||||||
enter the other example's final configuration.
|
|
||||||
|
|
||||||
Both selected roles must resolve to an existing profile, registered backend,
|
`glossary_collections` maps names to collections with an ordered `entries` list.
|
||||||
nonblank effective model, and nonblank endpoint. The normalized effective
|
Each entry has an `id`, `name`, and `category`; `description`, `aliases`, and
|
||||||
endpoint must match the application backend endpoint. A selected profile cannot
|
`plural` provide optional context. Entries are identified within their collection
|
||||||
introduce a competing `api_key_env` lookup; the application backend credential
|
by `id`. Root `glossaries` and `protected_terms` select collection names. If root
|
||||||
supplies the runtime key when required. Other unselected profiles may be
|
protection is omitted, it derives from the root glossary selection.
|
||||||
partial, but every final backend, profile, and prompt definition is validated
|
|
||||||
when the engine is constructed. A malformed or conflicting unused definition
|
|
||||||
can therefore make the selected root invalid. A missing selected root is an
|
|
||||||
operator configuration error; an empty root may contribute no definitions.
|
|
||||||
|
|
||||||
Profile IDs and parent IDs are exact, literal keys. PromptKit profiles can use
|
`pipeline.stages` is an ordered list of `module` records; supported modules are
|
||||||
native `extends` inheritance; child fields override inherited fields, while
|
`glossary`, `homophones`, `spoken_word`, and `grammar`. The default sequence is
|
||||||
unmentioned controls survive changes to model or backend. For example:
|
`glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`. Repeated modules
|
||||||
|
run separately; `[]` runs normalization without model processing. An optional
|
||||||
|
nonempty, unique `id` identifies a record in evidence. Each record may override
|
||||||
|
`glossaries`, `protected_terms`, `confidence_threshold`, and fields under `llm`,
|
||||||
|
`validation`, and `context`. Omitted fields inherit their resolved root values.
|
||||||
|
Explicit empty lists and null list selections clear a selection.
|
||||||
|
|
||||||
```yaml
|
Each entry receives its selected context and independent protection. Glossary
|
||||||
profiles:
|
correction exempts active recognized spellings from protection for that entry.
|
||||||
"shared~base":
|
A longer exempt name still takes precedence over a shorter protected spelling
|
||||||
backend: service
|
inside it. A glossary correction must have at least one selected entry at
|
||||||
model: example-model
|
processing preflight. Empty definitions and empty protection remain valid
|
||||||
temperature: 0.4
|
configuration.
|
||||||
top_p: 0.8
|
|
||||||
"proposal fast":
|
|
||||||
extends: "shared~base"
|
|
||||||
model: faster-model
|
|
||||||
temperature: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
Presence matters. Explicit `temperature: 0` is sent as zero; omission leaves it
|
|
||||||
unspecified; null suppresses an inherited temperature. Other fields have their
|
|
||||||
own native null and omission rules. Provider `extra_params` maps merge
|
|
||||||
recursively, and a null provider value is sent as JSON null. See the pinned
|
|
||||||
[PromptKit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/formats.md)
|
|
||||||
for the complete field semantics.
|
|
||||||
|
|
||||||
Operator documents can also override an embedded prompt/version, including its
|
|
||||||
messages and definition-level output settings. Shared variables supplied by the
|
|
||||||
adapter are rendered only when an effective message references them; Audita does
|
|
||||||
not append omitted shared text. Audita's per-call schemas, structural repair
|
|
||||||
budget, and downstream domain validators remain in force. The engine captures
|
|
||||||
effective definitions and message-file contents when constructed, so edits are
|
|
||||||
observed only by a newly constructed runtime.
|
|
||||||
|
|
||||||
### Migrating PromptKit profile documents
|
|
||||||
|
|
||||||
For the PromptKit v0.11 format, wrap each profile's fields under
|
|
||||||
`profiles: {<literal-id>: ...}`, remove inner `id`, and replace
|
|
||||||
`base_profile` with `extends`. Profile and prompt IDs are mapping keys, so they
|
|
||||||
need not match source filenames. Review zeros, omitted values, and nulls against
|
|
||||||
the native presence rules before deploying. Select a root whose final
|
|
||||||
definitions have valid backend references; eager validation applies even to
|
|
||||||
unselected definitions. See the complete maintained [local](../examples/minimal-config.yml)
|
|
||||||
and [service](../examples/production-config.yml) examples and the pinned
|
|
||||||
[consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
|
|
||||||
This incompatible profile-format migration is intended for an Audita major
|
|
||||||
release; no automatic file conversion is provided.
|
|
||||||
|
|
||||||
The [CLI inspection views](cli.md#streams-and-exit-status) show selected
|
|
||||||
controls and field provenance separately from ConfigKit's redacted settings
|
|
||||||
YAML. Inspection resolves both roles without provider calls.
|
|
||||||
|
|
||||||
## Settings
|
## Settings
|
||||||
|
|
||||||
| Field | Default | Rule |
|
| Setting | Default | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `profiles_dir` | `profiles` | Nonempty PromptKit document root; resolved relative to the selected main file, or absolute. |
|
| `llm.proposal_profile` | unset | PromptKit profile for proposals; required for processing entries. |
|
||||||
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
|
| `llm.validation_profile` | effective proposal profile | Profile for model validation; null clears a selection before fallback. |
|
||||||
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
|
| `llm.correction_repair_profile` | effective proposal profile | Profile for correction repair; null clears an inherited selection before fallback. |
|
||||||
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
|
| `llm.correction_repair_attempts` | `1` | Nonnegative repair attempts per correction; zero disables correction repair. |
|
||||||
| `backends.<id>.credential` | Unset | Optional environment-variable secret reference; omission or null means unauthenticated use. |
|
| `llm.transient_retries` | `2` | Additional transient attempts per generation. |
|
||||||
| `llm.proposal_profile` | None | Required nonempty profile ID. |
|
| `llm.repair_attempts` | `2` | Structural repairs, from zero through three. |
|
||||||
| `llm.validation_profile` | Final proposal profile | Omission inherits the final proposal selection; explicit empty or null fails. |
|
| `llm.operation_timeout` | `15m` | Positive duration per logical operation. |
|
||||||
| `llm.transient_retries` | `2` | Nonnegative additional attempts per generation. |
|
|
||||||
| `llm.repair_attempts` | `2` | Additional structural repair generations, from zero through three. |
|
|
||||||
| `llm.operation_timeout` | `15m` | Positive Go duration string per logical call. |
|
|
||||||
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module keys; repeats are meaningful and an explicit empty list runs no correction modules. |
|
|
||||||
| `output.schema` | `bare-segments` | `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md). |
|
|
||||||
| `validation.max_prompt_tokens` | `2048` | Positive validation batching budget. |
|
| `validation.max_prompt_tokens` | `2048` | Positive validation batching budget. |
|
||||||
| `chunking.target_sections` | Unset | Optional positive target; omission or null selects automatic chunking. |
|
| `thresholds.<module>` | `0.8` | Root confidence threshold for each module, in `[0, 1]`. |
|
||||||
| `chunking.max_section_tokens` | `8192` | Positive maximum estimate. |
|
| `context.description` | empty | Prompt context; at most 500 bytes after trimming. |
|
||||||
| `chunking.min_section_tokens` | `2048` | Positive and no greater than maximum. |
|
| `chunking.target_sections` | unset | Optional positive target. |
|
||||||
| `normalization.max_segment_gap` | `4.0` | Nonnegative numeric seconds. |
|
| `chunking.max_section_tokens`, `chunking.min_section_tokens` | `8192`, `2048` | Positive section bounds; minimum must not exceed maximum. |
|
||||||
| `normalization.ellipsis_gap` | `3.5` | Nonnegative numeric seconds. |
|
| `normalization.max_segment_gap`, `normalization.ellipsis_gap` | `4`, `3.5` | Nonnegative seconds. |
|
||||||
| `normalization.max_segment_duration` | `60.0` | Positive numeric seconds. |
|
| `normalization.max_segment_duration`, `normalization.max_segment_tokens` | `60`, `2048` | Positive seconds and token estimate. |
|
||||||
| `normalization.max_segment_tokens` | `2048` | Positive token estimate. |
|
| `output.schema` | `bare-segments` | Output format; see [output contracts](integrations/output-files.md). |
|
||||||
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
|
| `reporting.enabled`, `reporting.stdout_dir` | `true`, `.` | Ordinary report capture and default directory for stdout output. |
|
||||||
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
|
| `diagnostics.work_dir` | `/tmp/audita` | Parent directory for optional debug capture. |
|
||||||
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
|
|
||||||
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; a `--report-dir` flag can select reports for either transcript destination. |
|
|
||||||
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
|
|
||||||
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
|
|
||||||
|
|
||||||
Module keys select correction modules; validator chains remain built into them.
|
For example, an entry can disable correction repair or select another profile:
|
||||||
The loader preserves module order and repetitions. Normalization time settings
|
|
||||||
accept numeric seconds only. The operation timeout uses a duration string.
|
|
||||||
`reporting.stdout_dir` must be nonblank when it supplies the active report
|
|
||||||
destination: for stdout output, or for file output when `--report-dir` is
|
|
||||||
explicitly supplied. File output without that flag uses the transcript's parent,
|
|
||||||
so the configured value is inactive. Reporting-disabled runs do not select a
|
|
||||||
report directory. `diagnostics.work_dir` must be nonblank only with debug
|
|
||||||
enabled. These path requirements are checked when choosing process destinations,
|
|
||||||
not during configuration inspection.
|
|
||||||
|
|
||||||
## Credentials
|
```yaml
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
correction_repair_attempts: 2
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: grammar
|
||||||
|
llm:
|
||||||
|
correction_repair_profile: review
|
||||||
|
- module: spoken_word
|
||||||
|
llm:
|
||||||
|
correction_repair_attempts: 0
|
||||||
|
```
|
||||||
|
|
||||||
A backend credential is a ConfigKit secret reference containing an environment
|
Processing profiles are selected explicitly; no profile or glossary is selected
|
||||||
variable name, such as `credential: AUDITA_SERVICE_KEY`. Every reference that
|
automatically. The validation selection falls back after root and per-entry
|
||||||
survives overlay and override resolution must name a set, nonempty variable,
|
settings are combined. An explicit root correction repair profile is inherited
|
||||||
including credentials on backends unused by the selected roles. A replacement
|
by each entry unless overridden or cleared with null; otherwise the entry uses
|
||||||
reference resolves only the replacement; null removes the reference. General
|
its effective proposal profile. The repair profile is checked only when its
|
||||||
`AUDITA_*` and `OPENROUTER_API_KEY` variables have no implicit setting effect.
|
attempt budget is positive. Each correction repair attempt is a logical call
|
||||||
Secret values are available to the runtime through the resolved reference, while
|
that can also use the configured transient retries and structural repairs. Null
|
||||||
ConfigKit's effective output and provenance contain no resolved secret values.
|
does not disable validation. Numeric zero and false
|
||||||
|
retain their field meaning, subject to validation. Inspect recorded origins and
|
||||||
Current invocation syntax belongs in [CLI](cli.md).
|
default dependencies using the [native commands](cli.md).
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ ownership. Commit discipline and agent Git restrictions are in the
|
|||||||
|
|
||||||
## Baseline Validation
|
## Baseline Validation
|
||||||
|
|
||||||
Use Go 1.26.0 or newer for source builds. The selected release toolchain is
|
Use Go 1.27.0 or newer for source builds. The selected release toolchain is
|
||||||
documented in the [release procedure](release.md#prerequisites). From the
|
documented in the [release procedure](release.md#prerequisites). From the
|
||||||
repository root:
|
repository root:
|
||||||
|
|
||||||
|
|||||||
@@ -1,26 +1,26 @@
|
|||||||
# OpenAI-Compatible LLM Integration
|
# OpenAI-Compatible LLM Integration
|
||||||
|
|
||||||
Audita uses the pinned [PromptKit](https://gitea.maximumdirect.net/eric/promptkit)
|
Audita uses the pinned [PromptKit](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09)
|
||||||
built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing,
|
built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing,
|
||||||
provider envelope parsing, transient retries, per-generation timeouts, and JSON
|
provider envelope parsing, transient retries, per-generation timeouts, and JSON
|
||||||
Schema repair. A backend's configured base endpoint must support the selected
|
Schema repair. A backend's configured base endpoint must support the selected
|
||||||
profile's model and structured JSON-schema responses; local configuration
|
profile's model and structured JSON-schema responses; local configuration
|
||||||
inspection does not contact the provider.
|
inspection does not contact the provider.
|
||||||
|
|
||||||
Audita selects a role profile, supplies its registered backend credential, and
|
Audita selects role profiles independently for each pipeline entry and supplies
|
||||||
passes the embedded response schema as a complete per-call output contract. It
|
its domain response schema and per-operation budget. PromptKit owns backend and
|
||||||
prepares each call once, checks that the prepared target still matches the
|
credential selection, request preparation, transport, retries, repairs, and
|
||||||
inspected backend and endpoint, then executes that same prepared request. A
|
admission. Audita executes the same prepared handle and retains its own domain
|
||||||
profile cannot introduce a separate credential environment lookup. An absent
|
validation. Profiles may use native endpoint and credential settings; local
|
||||||
backend credential permits an unauthenticated local service.
|
inspection does not resolve a credential value or call the provider.
|
||||||
|
|
||||||
`llm.transient_retries` counts additional attempts for eligible transient
|
`llm.transient_retries` counts additional attempts for eligible transient
|
||||||
failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
||||||
additional generations for schema-invalid model output. The operation timeout
|
additional generations for schema-invalid model output. The operation timeout
|
||||||
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
||||||
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
||||||
for defaults and [PromptKit v0.14.0's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/consumers/pkg-promptkit.md)
|
for defaults and the [PromptKit consumer contract](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/consumers/pkg-promptkit.md)
|
||||||
for the pinned protocol and retry details.
|
for protocol and retry details.
|
||||||
|
|
||||||
A model operation that exhausts structural repair, a live-parent local operation
|
A model operation that exhausts structural repair, a live-parent local operation
|
||||||
timeout, capacity rejection, transport failure, malformed provider response,
|
timeout, capacity rejection, transport failure, malformed provider response,
|
||||||
@@ -31,4 +31,4 @@ continue. Caller cancellation, broken assets or configuration, and unknown
|
|||||||
internal failures remain fatal and cancel peers. Audita decodes into domain
|
internal failures remain fatal and cancel peers. Audita decodes into domain
|
||||||
types only after PromptKit validates the JSON. Ordinary reports use safe reason
|
types only after PromptKit validates the JSON. Ordinary reports use safe reason
|
||||||
codes and counts rather than provider messages. Full request/response content
|
codes and counts rather than provider messages. Full request/response content
|
||||||
is written only with `--debug`; see [operations](../operations.md#diagnostics-directory).
|
is written only with `--debug`; see [operations](../operations.md#debug-evidence).
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
The corrected transcript uses the selected `output.schema`: `bare-segments`
|
The corrected transcript uses the selected `output.schema`: `bare-segments`
|
||||||
or `audita-v1`. Its schema and text rules are unchanged by resilience
|
or `audita-v1`. Its schema and text rules are unchanged by resilience
|
||||||
reporting. See [configuration](../config.md#settings) for selection and
|
reporting. See [configuration](../config.md#settings) for selection and
|
||||||
[operations](../operations.md#required-output-files) for placement and writes.
|
[operations](../operations.md#outputs-and-lifecycle) for placement and writes.
|
||||||
|
|
||||||
With reporting enabled, Audita produces three ordinary files:
|
With reporting enabled, Audita produces three ordinary files:
|
||||||
|
|
||||||
@@ -46,9 +46,9 @@ remain in opt-in debug files.
|
|||||||
## Error and Recovery Records
|
## Error and Recovery Records
|
||||||
|
|
||||||
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
|
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
|
||||||
and `chunk_index`, module key/instance, initial segment range, applied-correction
|
and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction
|
||||||
count, and a `failures` sequence. Each failure identifies the operation
|
count, and a `failures` sequence. Each failure identifies the operation
|
||||||
(`proposal`, `validation`, or `validation_input`), optional validator and
|
(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
|
||||||
zero-based batch index, safe reason/message, sorted proposal indices when
|
zero-based batch index, safe reason/message, sorted proposal indices when
|
||||||
known, and execution counts when available. Multiple failed batches in one
|
known, and execution counts when available. Multiple failed batches in one
|
||||||
work unit form one error record. `scope: run` instead records phase, safe reason
|
work unit form one error record. `scope: run` instead records phase, safe reason
|
||||||
@@ -59,11 +59,15 @@ For `validation_decisions_invalid`, proposal indices identify only candidates
|
|||||||
whose required decisions were unavailable. Other candidates in the same batch
|
whose required decisions were unavailable. Other candidates in the same batch
|
||||||
may still be approved or rejected normally.
|
may still be approved or rejected normally.
|
||||||
|
|
||||||
Each recovery identifies its work unit and successful operation, with sorted
|
Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted
|
||||||
`reasons`, a safe message, available execution counts, and sorted
|
`reasons`, a safe message, available execution counts, and sorted
|
||||||
`filtered_proposals` details. One recovery can contain `output_repaired`,
|
`filtered_proposals` details. One recovery can contain `output_repaired`,
|
||||||
`service_recovered`, and `proposals_filtered` together. Filtering covers
|
`service_recovered`, `correction_revised`, and `proposals_filtered` together.
|
||||||
defective response proposals; low confidence, protected-term rejection,
|
`correction_revised` means a repair yielded a fully validated revision; its
|
||||||
|
application outcome is recorded in the Markdown report. Filtering covers
|
||||||
|
defective proposals and revisions under their originating operation. Only
|
||||||
|
original proposal operations count toward `proposal_responses_with_filtering`.
|
||||||
|
Low confidence, protected-term rejection,
|
||||||
ordinary editorial rejection, and an apply-time conflict caused by an earlier
|
ordinary editorial rejection, and an apply-time conflict caused by an earlier
|
||||||
accepted correction are not filtering recoveries. Failed operations are errors,
|
accepted correction are not filtering recoveries. Failed operations are errors,
|
||||||
not recoveries. The initial chunk identity remains stable even if earlier
|
not recoveries. The initial chunk identity remains stable even if earlier
|
||||||
@@ -79,23 +83,29 @@ is an error even when unexpected indices were also ignored.
|
|||||||
`report.md` includes run ID, status, corrections, work coverage, error and
|
`report.md` includes run ID, status, corrections, work coverage, error and
|
||||||
recovery totals, input/output paths, and full work-unit error and recovery
|
recovery totals, input/output paths, and full work-unit error and recovery
|
||||||
matrices. Rows show every initial chunk with 1-based chunk number and initial
|
matrices. Rows show every initial chunk with 1-based chunk number and initial
|
||||||
segment range; columns follow configured module-instance order, including
|
segment range; columns follow configured pipeline order, including repeated modules. Labels show
|
||||||
repeats. Completed error cells are `0` or `1`; recovery cells are operation
|
the one-based position and module, plus the operator ID when supplied. Completed error cells are `0` or `1`; recovery cells are operation
|
||||||
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
||||||
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
||||||
records containing each reason; a recovery can have several reasons, so those
|
records containing each reason; a recovery can have several reasons, so those
|
||||||
counts need not sum to the recovery total. The report refers to both YAML
|
counts need not sum to the recovery total. When a repair was attempted, a
|
||||||
files. It does not include transcript excerpts or correction lists.
|
Correction repair table lists its stage, chunk,
|
||||||
|
original proposal index, semantic attempt count, and final outcome (`applied`,
|
||||||
|
`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or
|
||||||
|
`interrupted`). The report refers to both YAML files. It does not include
|
||||||
|
transcript excerpts or correction lists.
|
||||||
|
|
||||||
## Debug-Only Files
|
## Debug-Only Files
|
||||||
|
|
||||||
With `--debug`, `correction-ledger.json` contains proposed candidates and
|
With `--debug`, `correction-ledger.json` contains proposed candidates and
|
||||||
their application, rejection, or interruption details. `llm-execution.json`
|
their application, rejection, or interruption details. `llm-execution.json`
|
||||||
holds logical calls and PromptKit execution events; `utilization-diagnostics.json`
|
holds logical calls and PromptKit execution events. Interaction request/response
|
||||||
groups backend capacity and timing observations while preserving known and
|
artifacts and input snapshots are also debug-only. Records carry zero-based
|
||||||
complete usage semantics. Interaction request/response artifacts, selected
|
`stage_index` and optional `stage_id` where an entry is identified.
|
||||||
profiles, configuration provenance, and input snapshots are also debug-only.
|
|
||||||
These files are diagnostic evidence, not an alternate ordinary report schema.
|
These files are diagnostic evidence, not an alternate ordinary report schema.
|
||||||
|
For a repaired proposal, one ledger row preserves the original proposed edit,
|
||||||
|
the final applied segment and span when applicable, and repair attempts with
|
||||||
|
revision decisions and interaction artifact paths.
|
||||||
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
||||||
literal replacement span, even when it is `""` for a deletion. Applied entries
|
literal replacement span, even when it is `""` for a deletion. Applied entries
|
||||||
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
||||||
|
|||||||
@@ -6,15 +6,15 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
|||||||
|
|
||||||
## Invocation
|
## Invocation
|
||||||
|
|
||||||
Use an argument array with the transcript path and required glossary argument,
|
Use an argument array with the transcript path and configured glossary selections,
|
||||||
following the [process examples](../cli.md#commands-and-help). For machine workflows,
|
following the [process examples](../cli.md#processing). For machine workflows,
|
||||||
select an explicit transcript file destination and a configuration path. Reports
|
select an explicit transcript file destination and a configuration path. Reports
|
||||||
go beside the transcript unless an explicit `--report-dir` selects another
|
go beside the transcript unless an explicit `--report-dir` selects another
|
||||||
directory. For stdout output, reports use the effective `reporting.stdout_dir`
|
directory. For stdout output, reports use the effective `reporting.stdout_dir`
|
||||||
(defaulting to the invocation working directory), including a `--report-dir`
|
(defaulting to the invocation working directory), including a `--report-dir`
|
||||||
override. A config-file or overlay value for `reporting.stdout_dir` alone does
|
override. A config-file or overlay value for `reporting.stdout_dir` alone does
|
||||||
not redirect file-output reports. `--work-dir` selects a debug root, while
|
not redirect file-output reports. `--work-dir` selects a debug root, while
|
||||||
configuration or `--debug` enables capture; the path flag alone does not.
|
`--debug` enables capture; the path flag alone does not.
|
||||||
Supplying a configuration path also makes file selection predictable; only
|
Supplying a configuration path also makes file selection predictable; only
|
||||||
explicitly referenced credential variables are read from the environment.
|
explicitly referenced credential variables are read from the environment.
|
||||||
|
|
||||||
@@ -26,19 +26,20 @@ or cancellation in the parent process when needed.
|
|||||||
## Result Handling
|
## Result Handling
|
||||||
|
|
||||||
Interpret the exit result according to the
|
Interpret the exit result according to the
|
||||||
[CLI output and exit contract](../cli.md#streams-and-exit-status).
|
[CLI output and exit contract](../cli.md#inspection-and-outcomes).
|
||||||
Consume transcript/report data through the
|
Consume transcript/report data through the
|
||||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||||
The three reports may be absent after early configuration or destination
|
Audita loads configuration and parses the input transcript before report
|
||||||
failures. After initialization, failed reports can describe invalid input or
|
initialization. Early destination failures can also leave no reports. After
|
||||||
partial processing and use the same selected report directory as successful
|
initialization, failed reports can describe partial processing or output
|
||||||
runs. Exit `0` can include recoverable model errors and applied
|
failures and use the same report directory as successful runs.
|
||||||
|
Exit `0` can include recoverable model errors and applied
|
||||||
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
||||||
transcript may have been written before a later required report failure, so
|
transcript may have been written before a later required report failure, so
|
||||||
the file alone does not prove completion.
|
the file alone does not prove completion.
|
||||||
|
|
||||||
If processing fails, preserve available evidence and follow
|
If processing fails, preserve available evidence and follow
|
||||||
[operations recovery](../operations.md#failure-inspection-and-recovery).
|
[operations recovery](../operations.md#recovery).
|
||||||
Ordinary reports are independent of optional retained debug evidence. Parent
|
Ordinary reports are independent of optional retained debug evidence. Parent
|
||||||
cancellation reaches the child through its context; Audita joins workers and
|
cancellation reaches the child through its context; Audita joins workers and
|
||||||
attempts failed-status report finalization before exiting. Uncatchable
|
attempts failed-status report finalization before exiting. Uncatchable
|
||||||
@@ -49,4 +50,4 @@ file nor a debug directory alone establishes successful completion.
|
|||||||
|
|
||||||
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
||||||
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
||||||
- [Configuration inspection commands](../cli.md#commands-and-help)
|
- [Configuration inspection commands](../cli.md#inspection-and-outcomes)
|
||||||
|
|||||||
@@ -28,23 +28,11 @@ The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyabl
|
|||||||
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
||||||
types, duplicate supplied IDs, and the validation failures above are rejected.
|
types, duplicate supplied IDs, and the validation failures above are rejected.
|
||||||
|
|
||||||
## Glossary YAML Contract
|
## Glossary collections
|
||||||
|
|
||||||
The top-level object contains a `glossary` list with at least one entry.
|
Glossary definitions are composed as application configuration, including the
|
||||||
|
optional companion inputs described by the [configuration reference](../config.md).
|
||||||
| Entry field | Type and validation |
|
The [local example](../../examples/local/glossary.yml) provides a complete
|
||||||
| --- | --- |
|
collection. Configuration may contain empty collections; processing preflight
|
||||||
| `name` | Non-empty string |
|
requires every glossary-correction entry to select at least one glossary entry.
|
||||||
| `category` | Non-empty string |
|
The previous standalone `glossary` file format is not a processing input.
|
||||||
| `summary` | Non-empty string |
|
|
||||||
| `aliases` | Optional list of non-empty strings |
|
|
||||||
| `plural` | Optional string |
|
|
||||||
|
|
||||||
The parser checks empty strings without trimming them. Unknown YAML fields are
|
|
||||||
ignored by the glossary parser, unlike the stricter
|
|
||||||
[configuration-file schema](../config.md#loading-and-precedence).
|
|
||||||
|
|
||||||
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
|
|
||||||
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
|
|
||||||
and empty aliases are rejected. These formats describe input acceptance, not
|
|
||||||
additional vocabulary restrictions.
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
|
|||||||
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
||||||
2. Update file, selected overlay, credential reference, and CLI mapping as
|
2. Update file, selected overlay, credential reference, and CLI mapping as
|
||||||
applicable. Preserve the precedence defined in the
|
applicable. Preserve the precedence defined in the
|
||||||
[configuration contract](../config.md#loading-and-precedence), including the
|
[configuration contract](../config.md#loading-and-composition), including the
|
||||||
distinct validation and effective-config commands.
|
distinct validation and effective-config commands.
|
||||||
3. Update redaction when secrets are involved.
|
3. Update redaction when secrets are involved.
|
||||||
4. Review configuration tests and related CLI tests for precedence, validation,
|
4. Review configuration tests and related CLI tests for precedence, validation,
|
||||||
@@ -29,33 +29,18 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
|
|||||||
|
|
||||||
The `version` route uses ReleaseKit's public `buildinfo.Version()` accessor and
|
The `version` route uses ReleaseKit's public `buildinfo.Version()` accessor and
|
||||||
stays independent of application runtime assembly. Preserve its behavior from
|
stays independent of application runtime assembly. Preserve its behavior from
|
||||||
the [CLI contract](../cli.md#commands-and-help) when changing the command; the
|
the [CLI contract](../cli.md) when changing the command; the
|
||||||
adoption decision is recorded in [ADR 0001](../adr/0001-releasekit.md).
|
adoption decision is recorded in [ADR 0001](../adr/0001-releasekit.md).
|
||||||
|
|
||||||
## Validating Examples
|
## Validating Examples
|
||||||
|
|
||||||
Keep complete copyable files in [examples](../../examples/) and link to them from
|
Keep complete copyable files in [examples](../../examples/) and link to them from
|
||||||
their canonical contract documentation. Keep them valid and secret-free.
|
their canonical contract documentation. Keep them valid and secret-free.
|
||||||
From the repository root, validate the maintained configuration examples:
|
Run the commands in the [example guide](../../examples/README.md) from each
|
||||||
|
project's directory. `internal/cli/examples_integration_test.go` exercises the
|
||||||
```sh
|
maintained projects with a controlled provider, and the preview fixture can be
|
||||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
prepared without inference. Review automatic companion sources and overlays in
|
||||||
go run ./cmd/audita config validate --config examples/configkit-inheritance.yml
|
their actual project directories. Keep service credentials synthetic in tests.
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
|
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
|
|
||||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
|
|
||||||
go run ./cmd/audita config print-effective --config examples/configkit-inheritance.yml --view explain
|
|
||||||
```
|
|
||||||
|
|
||||||
The local and ConfigKit inheritance files select `profiles/local`; the service
|
|
||||||
file selects `profiles/service`, including the service testing profiles. Keep
|
|
||||||
these roots separate because PromptKit validates every final definition in a
|
|
||||||
selected root, including definitions that are not selected for a role.
|
|
||||||
|
|
||||||
Only referenced credentials read environment values; use a controlled synthetic
|
|
||||||
value when validating the service example. Check affected CLI examples against the
|
|
||||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
|
||||||
validation should not invoke a live LLM merely to check documentation.
|
|
||||||
|
|
||||||
## Runtime Change Checks
|
## Runtime Change Checks
|
||||||
|
|
||||||
|
|||||||
@@ -24,36 +24,29 @@ retain their execution counts and never produce a recovery record.
|
|||||||
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
||||||
transcript preflight, report initialization, regular-file replacement, and direct
|
transcript preflight, report initialization, regular-file replacement, and direct
|
||||||
stream publication. Transcript input is protected from early report writes but
|
stream publication. Transcript input is protected from early report writes but
|
||||||
may be the final transcript destination. The LLM runtime records successful reads
|
may be the final transcript destination. The LLM runtime records consumed PromptKit sources and effective host message
|
||||||
through PromptKit's profile filesystem and supplies those source paths to CLI;
|
files and supplies those source paths to CLI;
|
||||||
no second filename-based discovery rule is used.
|
no second filename-based discovery rule is used.
|
||||||
CLI composes it after configuration/profile inspection and before reading
|
CLI composes it after configuration, transcript acquisition, and preflight
|
||||||
inputs or dispatching model work. It does not create ordinary report parents.
|
readiness and before dispatching model work. It does not create ordinary report parents.
|
||||||
On a final write error it attempts other enabled reports and one failed-status
|
On a final write error it attempts other enabled reports and one failed-status
|
||||||
pass; a published transcript is never erased by a later report failure.
|
pass; a published transcript is never erased by a later report failure.
|
||||||
|
|
||||||
Debug is independent. Only `diagnostics.debug` creates a unique retained child
|
Debug is independent. The shared `--debug` flag enables a unique retained child
|
||||||
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
|
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
|
||||||
when debug is off. With debug on, source/normalized snapshots, invocation,
|
when debug is off. With debug enabled, it captures source and normalized
|
||||||
effective settings, origins, profiles, initial chunk summary, correction ledger,
|
snapshots, invocation, effective settings, correction ledger, LLM execution
|
||||||
indexed execution/utilization, and model interactions are written. Runner
|
observations, and model interactions. Candidate capture includes proposals later
|
||||||
candidate capture includes proposals later rejected or interrupted. Ledger
|
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
|
||||||
mapping uses optional string pointers so an empty proposed replacement is
|
attempt, validator, and batch coordinates as applicable; their retained content
|
||||||
serialized for every candidate and an empty applied replacement appears only
|
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
|
||||||
when that candidate applied; see the [public artifact contract](../integrations/output-files.md#debug-only-files).
|
replacements and distinguishes unavailable candidates from interrupted work.
|
||||||
Debug write failures reach the CLI through an explicit concurrency-safe diagnostic
|
|
||||||
callback and produce redacted stderr messages without aborting processing.
|
A concurrency-safe diagnostic callback reports optional write failures without
|
||||||
Completed validator decisions returned with a fatal batch error remain available
|
aborting processing. Diagnostic redaction uses credential source metadata; debug
|
||||||
to the debug ledger; they never authorize applying an incompletely validated edit.
|
artifacts can still contain sensitive transcript and prompt content. Execution
|
||||||
Runner decision records carry the validator's explicit approved, rejected, or
|
position and optional operator ID are carried separately from generated safe
|
||||||
unavailable disposition into the ledger. Unavailable candidates have a distinct
|
artifact path coordinates.
|
||||||
top-level ledger disposition, while fatal interruptions remain interrupted when
|
|
||||||
no rejection or unavailable decision was returned. Successful validation
|
|
||||||
interaction payloads contain the full decoded response, final content, and
|
|
||||||
reconciliation outcomes; ignored indices do not become ledger candidates.
|
|
||||||
Every debug writer uses all configured backend secrets for redaction, including
|
|
||||||
unselected backend credentials and source snapshots. Debug can still contain
|
|
||||||
sensitive transcript and prompt content.
|
|
||||||
|
|
||||||
## Key Tests
|
## Key Tests
|
||||||
|
|
||||||
|
|||||||
@@ -1,60 +1,30 @@
|
|||||||
# Audita LLM Runtime
|
# Audita LLM Runtime
|
||||||
|
|
||||||
## Client Boundary
|
`llm.NewRuntime` assembles one captured PromptKit engine using its default
|
||||||
|
catalog, Audita's embedded prompts and schemas, and optional operator sources
|
||||||
|
anchored to the recorded application main context. Captured host sources and
|
||||||
|
message files are passed to output collision checks. An embedded-only diagnostic
|
||||||
|
construction distinguishes shipped asset failures from operator-source failures;
|
||||||
|
it never replaces the requested engine.
|
||||||
|
|
||||||
`llm.NewRuntime` assembles one PromptKit engine for both roles. PromptKit's
|
Each resolved pipeline entry receives role-bound clients with its selected
|
||||||
catalogs and defaults are followed by Audita's embedded prompt and schema
|
profiles and effective budgets. Clients share engine-local backend admission.
|
||||||
inputs, operator documents from `profiles_dir`, and the highest-priority
|
The adapter uses `PrepareExecution` and `RunPrepared` on the same handle, passes
|
||||||
application backend mapping. Audita owns backend endpoints, capacity settings,
|
Audita's per-call schema and repair budget, and decodes only validated results.
|
||||||
and resolved credentials; native PromptKit layering owns profile inheritance,
|
PromptKit owns backend and credential selection. The adapter uses captured
|
||||||
prompt overrides, and provider-specific fields. Imports are disabled. The
|
credential source names for artifact redaction without inspecting unrelated
|
||||||
runtime inspects both selected roles offline, requiring complete model and
|
environment values. See the [integration contract](../integrations/openai-compatible-llm.md)
|
||||||
endpoint values and rejecting competing profile credential lookups. A
|
for failure behavior and the [configuration reference](../config.md) for sources.
|
||||||
role-bound `llm.RoleClient` implements `structuredoutput.Client`; proposal and
|
|
||||||
validator contracts alias that interface. Both roles share engine-local backend
|
|
||||||
capacity.
|
|
||||||
|
|
||||||
Successful construction captures the final definitions and referenced message
|
When correction repair is enabled, the proposal call retains its prompt,
|
||||||
contents. The runtime does not reload operator documents or message files during
|
effective variables, prepared messages, and full final response for section-local
|
||||||
calls. If final engine construction fails, an embedded-only diagnostic
|
replay. The adapter renders the repair definition with the selected profile,
|
||||||
construction distinguishes shipped asset failures from invalid operator input.
|
appends only that branch’s history, and checks the original prepared prefix
|
||||||
An asset failure remains an asset error; when embedded construction succeeds,
|
before execution. The complete conversation is sent to the selected provider;
|
||||||
the original failure remains a configuration error. The diagnostic engine
|
provider rejection ends that branch as unavailable. The requested repair
|
||||||
never replaces or executes the requested configuration, and caller cancellation
|
definition is retained in call attribution alongside native base-prompt metadata.
|
||||||
skips this path. The captured inspection also supplies an internal source
|
|
||||||
ledger: all loaded host YAML plus effective, source-relative host
|
|
||||||
`content_file` resources are tracked for output-collision checks.
|
|
||||||
|
|
||||||
For each logical call, the adapter starts an operation context and a diagnostic
|
The observation collector tracks logical calls and execution events. Ordinary
|
||||||
record before preparation. It supplies the reserved hardening text, the selected
|
reports use compact safe outcomes; request and response evidence is optional
|
||||||
profile and resolved backend credential, and the complete embedded schema/repair
|
debug material. The runner preserves ordered pipeline traversal and deterministic
|
||||||
contract. `PrepareExecution` produces the exact messages and hashes retained in
|
section collection while canceling and joining peers on fatal errors.
|
||||||
an Audita snapshot. The adapter checks the prepared backend and endpoint before
|
|
||||||
`RunPrepared`. It records available final content, usage, model, and profile
|
|
||||||
metadata and decodes only a validated result. Call hooks expose logical
|
|
||||||
start/finish records and PromptKit execution events for debug utilization.
|
|
||||||
|
|
||||||
The [integration contract](../integrations/openai-compatible-llm.md) describes
|
|
||||||
wire ownership and failure behavior. `structuredoutput.Failure` has a safe
|
|
||||||
message, fixed reason code, typed cause, and recoverable/fatal/canceled class.
|
|
||||||
Counts distinguish provider attempts, transient retries, and structural
|
|
||||||
repairs, including on errors. Secrets from all configured backends, including unselected
|
|
||||||
ones, are available to the diagnostics redaction boundary.
|
|
||||||
|
|
||||||
CLI explain output and debug configuration origins share a value-free projection
|
|
||||||
of effective field origins. The [CLI contract](../cli.md) defines its optional
|
|
||||||
inheritance evidence fields.
|
|
||||||
|
|
||||||
## Orchestration
|
|
||||||
|
|
||||||
CLI processing supplies two role clients from the same engine to the runner.
|
|
||||||
PromptKit's per-backend admission applies to both roles and every generation,
|
|
||||||
retry, and repair. The runner still processes module instances in order,
|
|
||||||
collects concurrent section work deterministically, and cancels remaining
|
|
||||||
section work only on fatal failures. `llm.DiagnosticsWriter` remains the protected
|
|
||||||
request/response artifact writer. `llm.ObservationCollector` captures metadata
|
|
||||||
events under a short mutex and freezes them after runner work joins. Its logical
|
|
||||||
records include preparation failures; summary usage comes from terminal events
|
|
||||||
once per call and retains knownness and completeness for each field. Runner wall
|
|
||||||
and validator timings remain separate from provider attempts. Ordinary reports
|
|
||||||
derive compact runner outcomes; full observations are debug-only.
|
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ Modules implement `contracts.TranscriptModule`:
|
|||||||
- `Validators()`
|
- `Validators()`
|
||||||
- `Propose(ctx, req)`
|
- `Propose(ctx, req)`
|
||||||
|
|
||||||
|
Built-in modules also expose `RepairPromptID()` for the runner to select their
|
||||||
|
repair definitions; proposal-only callers retain the existing interface.
|
||||||
|
|
||||||
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
||||||
|
|
||||||
## Built-In Modules
|
## Built-In Modules
|
||||||
@@ -37,9 +40,9 @@ Shared proposal-generation plumbing is centralized in:
|
|||||||
Module packages own:
|
Module packages own:
|
||||||
- prompt selection (`internal/prompts` prompt IDs).
|
- prompt selection (`internal/prompts` prompt IDs).
|
||||||
|
|
||||||
`proposal_generation.BuildProposalVariables` encodes the glossary, selected
|
Each stage prepares the encoded glossary context and protected spellings once.
|
||||||
transcript section, and trimmed description once for all four modules. PromptKit
|
Proposal generation adds the selected transcript section and trimmed description
|
||||||
renders their embedded definitions from those variables and validates the
|
for each request. PromptKit renders the selected definition and validates the
|
||||||
correction-set schema before Audita maps proposals. Domain validators still
|
correction-set schema before Audita maps proposals. Domain validators still
|
||||||
check confidence, applicability, protected glossary terms, and replacement
|
check confidence, applicability, protected glossary terms, and replacement
|
||||||
policy. Model-facing segment IDs are one-based positions within the proposal
|
policy. Model-facing segment IDs are one-based positions within the proposal
|
||||||
@@ -61,6 +64,10 @@ Module packages call the built-in chain resolver at construction.
|
|||||||
|
|
||||||
## Failure and Outcome Behavior
|
## Failure and Outcome Behavior
|
||||||
|
|
||||||
|
Each actionable rejection can be revised or withdrawn within its original
|
||||||
|
section. A revision retains the original proposal identity and must pass the
|
||||||
|
full validator chain before application.
|
||||||
|
|
||||||
Module construction failures are fatal. Recoverable proposal model failures
|
Module construction failures are fatal. Recoverable proposal model failures
|
||||||
record an unsuccessful operation and yield no candidates for that section;
|
record an unsuccessful operation and yield no candidates for that section;
|
||||||
other sections and module instances continue. Fatal adapter or application
|
other sections and module instances continue. Fatal adapter or application
|
||||||
@@ -88,4 +95,4 @@ outcomes are defined by the [report contract](../integrations/output-files.md).
|
|||||||
- `internal/modules/*/module_test.go`
|
- `internal/modules/*/module_test.go`
|
||||||
- `internal/framework/modules/registry_test.go`
|
- `internal/framework/modules/registry_test.go`
|
||||||
- `internal/framework/proposal_generation/*_test.go`
|
- `internal/framework/proposal_generation/*_test.go`
|
||||||
- `internal/cli/run_test.go` (pipeline/report integration)
|
- `internal/cli/process_ordered_integration_test.go` (pipeline/report integration)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ selectable values and defaults belong to
|
|||||||
|
|
||||||
Configuration validation checks registry support. CLI orchestration resolves the
|
Configuration validation checks registry support. CLI orchestration resolves the
|
||||||
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
||||||
to the destination defined by the [CLI contract](../cli.md#streams-and-exit-status).
|
to the destination defined by the [CLI contract](../cli.md#inspection-and-outcomes).
|
||||||
An unsupported key fails validation or resolution rather than selecting an
|
An unsupported key fails validation or resolution rather than selecting an
|
||||||
implicit fallback.
|
implicit fallback.
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ Executable and command orchestration:
|
|||||||
|
|
||||||
Core deterministic components:
|
Core deterministic components:
|
||||||
- `internal/core/config`
|
- `internal/core/config`
|
||||||
|
- `internal/core/glossary`
|
||||||
- `internal/core/schema`
|
- `internal/core/schema`
|
||||||
- `internal/core/normalization`
|
- `internal/core/normalization`
|
||||||
- `internal/core/chunking`
|
- `internal/core/chunking`
|
||||||
@@ -24,6 +25,7 @@ Core deterministic components:
|
|||||||
|
|
||||||
Framework orchestration and contracts:
|
Framework orchestration and contracts:
|
||||||
- `internal/framework/contracts`
|
- `internal/framework/contracts`
|
||||||
|
- `internal/framework/preflight`
|
||||||
- `internal/framework/modules`
|
- `internal/framework/modules`
|
||||||
- `internal/framework/proposal_generation`
|
- `internal/framework/proposal_generation`
|
||||||
- `internal/framework/proposals`
|
- `internal/framework/proposals`
|
||||||
@@ -66,7 +68,7 @@ artifact lifecycle.
|
|||||||
## Component Responsibilities
|
## Component Responsibilities
|
||||||
|
|
||||||
Core components implement configuration defaults/loading/validation/redaction,
|
Core components implement configuration defaults/loading/validation/redaction,
|
||||||
transcript and glossary schemas, normalization and chunking, output encoding,
|
transcript schemas and derived glossary views, normalization and chunking, output encoding,
|
||||||
diagnostics artifact helpers, and public report structures.
|
diagnostics artifact helpers, and public report structures.
|
||||||
|
|
||||||
Framework components implement module and validator contracts, proposal
|
Framework components implement module and validator contracts, proposal
|
||||||
|
|||||||
@@ -4,8 +4,9 @@ The [input contract](../integrations/transcript-glossary-files.md) owns input
|
|||||||
formats; [configuration](../config.md) owns settings; the
|
formats; [configuration](../config.md) owns settings; the
|
||||||
[output contract](../integrations/output-files.md) owns serialized results.
|
[output contract](../integrations/output-files.md) owns serialized results.
|
||||||
|
|
||||||
1. CLI filesystem adapters read transcript and glossary files. Schema parsing
|
1. CLI loads resolved configuration and transcript input. Preflight checks all
|
||||||
produces source objects; transcript segments enter canonical normalization.
|
selected glossary views and roles before the output session prepares resources.
|
||||||
|
Transcript segments then enter canonical normalization.
|
||||||
2. Normalization sorts and merges segments using configured time and token
|
2. Normalization sorts and merges segments using configured time and token
|
||||||
limits, reassigning IDs where needed.
|
limits, reassigning IDs where needed.
|
||||||
3. The runner partitions the normalized transcript exactly once before its
|
3. The runner partitions the normalized transcript exactly once before its
|
||||||
@@ -18,10 +19,14 @@ formats; [configuration](../config.md) owns settings; the
|
|||||||
may preserve a checked subset of completed decisions as diagnostic evidence.
|
may preserve a checked subset of completed decisions as diagnostic evidence.
|
||||||
Recoverable failed batches mark their candidates unavailable; successful
|
Recoverable failed batches mark their candidates unavailable; successful
|
||||||
survivors continue through all required validators.
|
survivors continue through all required validators.
|
||||||
6. Only fully approved proposals reach application. Corrections apply in
|
6. Actionable rejections can enter independent section-local repair branches.
|
||||||
|
Each revision is mapped to the original section and revalidated from the
|
||||||
|
start. Withdrawal, exhaustion, and recoverable failures finish the branch
|
||||||
|
without affecting independent approvals.
|
||||||
|
7. Only fully approved proposals reach application. Corrections apply in
|
||||||
proposal-index order within their owning section. Existing replacement
|
proposal-index order within their owning section. Existing replacement
|
||||||
policies and apply-time safety checks resolve stale conflicts.
|
policies and apply-time safety checks resolve stale conflicts.
|
||||||
7. CLI encodes the final transcript and publishes active files through the
|
8. CLI encodes the final transcript and publishes active files through the
|
||||||
output session. `framework/processreport` builds one snapshot for YAML,
|
output session. `framework/processreport` builds one snapshot for YAML,
|
||||||
Markdown, and CLI totals. Debug evidence is captured separately when enabled.
|
Markdown, and CLI totals. Debug evidence is captured separately when enabled.
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,43 @@
|
|||||||
# Audita Prompt Registry
|
# Audita Prompt Registry
|
||||||
|
|
||||||
`internal/prompts` embeds seven PromptKit YAML definitions under
|
`internal/prompts` embeds eleven PromptKit definitions in the `audita`
|
||||||
`assets/prompts`. The IDs and selected version remain:
|
collection. The four proposal IDs are `audita/modules.glossary.proposal`,
|
||||||
|
`audita/modules.homophones.proposal`, `audita/modules.spoken_word.proposal`,
|
||||||
|
and `audita/modules.grammar.proposal`; the corresponding repair IDs replace
|
||||||
|
`.proposal` with `.repair`. The three validator IDs are
|
||||||
|
`audita/validators.spoken_form_plausibility`,
|
||||||
|
`audita/validators.meaning_reversal_review`, and
|
||||||
|
`audita/validators.editorial_review`.
|
||||||
|
|
||||||
- `modules.glossary.proposal` (`v1`)
|
PromptKit composes embedded defaults with operator prompt sources. Shared
|
||||||
- `modules.homophones.proposal` (`v1`)
|
instructions come from the embedded fragments in `assets/shared`; the request
|
||||||
- `modules.spoken_word.proposal` (`v1`)
|
shared-variable helper fills absent `HardeningPolicy`, `ProposalContract`,
|
||||||
- `modules.grammar.proposal` (`v1`)
|
`RepairContract`, and `ValidationContract` variables. The processing adapter
|
||||||
- `validators.spoken_form_plausibility` (`v1`)
|
supplies the embedded instruction fragments for those names. Processing supplies
|
||||||
- `validators.meaning_reversal_review` (`v1`)
|
Audita's response-schema contract separately.
|
||||||
- `validators.editorial_review` (`v1`)
|
The operator's prompt definition controls which variables its template uses.
|
||||||
|
|
||||||
|
Proposal focus and validator acceptance intentionally differ: minor editorial
|
||||||
|
cleanup may be accepted even when the proposing module was not asked to perform
|
||||||
|
it. Follow the [architecture boundary](../policy/architecture.md#correction-and-concurrency-invariants)
|
||||||
|
when maintaining proposal, validation, and repair instructions; the difference
|
||||||
|
alone is not a reason to tighten validation. The testing rationale is recorded in
|
||||||
|
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md).
|
||||||
|
|
||||||
|
For a standard request preview or validation, select a native PromptKit profile
|
||||||
|
and supply the template variables used by the selected prompt. The shipped
|
||||||
|
proposal definitions use `TranscriptDescriptionBlock`, `GlossaryJSON`,
|
||||||
|
`ProtectedTermsJSON`, and `SectionJSON`. Glossary entries provide context,
|
||||||
|
and the separate spelling list defines protection. Preview defaults the
|
||||||
|
spelling list to `[]` when omitted. The validator definitions use `TranscriptDescriptionBlock` and
|
||||||
|
`PayloadJSON`. The latter is supplied candidate JSON and does not require a
|
||||||
|
proposal inference call. Repair definitions use `RepairContract` and
|
||||||
|
`RepairRequestJSON`, containing the original proposal reference, current
|
||||||
|
correction, and ordered findings. An operator replacement may use different
|
||||||
|
variables.
|
||||||
|
See the [upstream CLI reference](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/cli.md)
|
||||||
|
for the standard preview, validation, and smoke commands.
|
||||||
|
|
||||||
PromptKit reads the embedded definitions as shipped defaults and renders their
|
|
||||||
messages from named string variables. Operator documents in `profiles_dir` may
|
|
||||||
contribute native keyed prompt definitions and override an embedded
|
|
||||||
prompt/version, including its messages and definition-level output settings.
|
|
||||||
The effective operator text is captured when the runtime is constructed; the
|
|
||||||
embedded assets remain the source of Audita's shipped task instructions. The
|
|
||||||
role-bound adapter supplies
|
|
||||||
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
|
|
||||||
embedded fragments under `assets/shared`, after caller variables so callers
|
|
||||||
cannot replace those instructions. Common literal-span and identity guidance
|
|
||||||
lives in the proposal fragment; shared candidate-reference, context, and
|
|
||||||
confidence guidance lives in the validation fragment. Module-specific policies
|
|
||||||
and examples remain in YAML. Proposal prompts receive the section JSON, glossary
|
|
||||||
JSON where used, and transcript description block. Validator prompts receive
|
|
||||||
the correction payload and description block.
|
|
||||||
The embedded prompt assets retain the shipped task instructions and
|
|
||||||
output-schema association. An operator override controls which template
|
|
||||||
variables its messages reference: supplying a shared variable does not append
|
|
||||||
its text when the effective message omits that reference. The role-bound runtime
|
|
||||||
supplies the complete JSON Schema output contract and repair count; the
|
|
||||||
definition does not own an operator-facing repair default.
|
|
||||||
The correction-set schema requires a string replacement and accepts an empty
|
The correction-set schema requires a string replacement and accepts an empty
|
||||||
string. Proposal guidance uses it for eligible span deletion and permits literal
|
string. Proposal guidance uses it for eligible span deletion and permits literal
|
||||||
spacing edits; validator guidance evaluates the full before/after segment under
|
spacing edits; validator guidance evaluates the full before/after segment under
|
||||||
@@ -39,14 +45,14 @@ each validator's existing remit. A blank resulting segment remains a domain
|
|||||||
rejection, not a structural schema error.
|
rejection, not a structural schema error.
|
||||||
|
|
||||||
`internal/framework/responseschema` remains the only embedded source for
|
`internal/framework/responseschema` remains the only embedded source for
|
||||||
`correction_set.v1.json` and `validator_decision_set.v1.json`. It exposes that
|
`correction_set.v1.json`, `correction_repair.v1.json`, and
|
||||||
|
`validator_decision_set.v1.json`. It exposes that
|
||||||
filesystem to PromptKit and retains Audita's schema-key lookup. PromptKit's
|
filesystem to PromptKit and retains Audita's schema-key lookup. PromptKit's
|
||||||
prepared execution supplies definition and rendered hashes; Audita does not
|
prepared execution supplies definition and rendered hashes; Audita does not
|
||||||
recompute or promise equality with hashes from the removed template renderer.
|
recompute or promise equality with hashes from the removed template renderer.
|
||||||
The small Audita registry records only prompt ID, version, source, and embedded
|
The small Audita registry records only prompt ID, source, and embedded path. Prepared runtime metadata belongs in diagnostics rather than the registry.
|
||||||
path. Prepared runtime metadata belongs in diagnostics rather than the registry.
|
|
||||||
|
|
||||||
The structured-call contract carries prompt ID/version, string variables,
|
The structured-call contract carries prompt ID, string variables,
|
||||||
response-schema key, and logical-call attribution. Module and validator callers
|
response-schema key, and logical-call attribution. Module and validator callers
|
||||||
pass these values through the role-bound adapter. See the
|
pass these values through the role-bound adapter. See the
|
||||||
[LLM runtime](llm-runtime.md) for runtime assembly and
|
[LLM runtime](llm-runtime.md) for runtime assembly and
|
||||||
|
|||||||
@@ -46,6 +46,14 @@ For each module section:
|
|||||||
3. record decisions and compact operation outcomes;
|
3. record decisions and compact operation outcomes;
|
||||||
4. carry only approved proposals forward.
|
4. carry only approved proposals forward.
|
||||||
|
|
||||||
|
When correction repair is enabled, the runner also gathers independent findings
|
||||||
|
for rejected candidates when a validator has usable evidence. Unavailable
|
||||||
|
assessments remain unavailable, and only candidates that pass every applicable
|
||||||
|
check can advance. A known rejection can enter repair even when another
|
||||||
|
assessment is unavailable; an unavailable assessment alone does not. Revised
|
||||||
|
candidates run the applicable checks again. A zero repair budget retains the
|
||||||
|
original shortlist flow.
|
||||||
|
|
||||||
Decision cardinality is enforced inside Audita: each candidate proposal must
|
Decision cardinality is enforced inside Audita: each candidate proposal must
|
||||||
receive exactly one outcome per validator it reaches. The runner still rejects
|
receive exactly one outcome per validator it reaches. The runner still rejects
|
||||||
duplicate, missing, or unknown identities in an internal validator result.
|
duplicate, missing, or unknown identities in an internal validator result.
|
||||||
@@ -116,19 +124,21 @@ is covered by [report assembly](diagnostics-reporting.md).
|
|||||||
- `internal/validators/*_test.go`
|
- `internal/validators/*_test.go`
|
||||||
- `internal/framework/validators/*_test.go`
|
- `internal/framework/validators/*_test.go`
|
||||||
- `internal/framework/processreport/correction_ledger_test.go`
|
- `internal/framework/processreport/correction_ledger_test.go`
|
||||||
- `internal/cli/run_test.go`
|
- `internal/cli/process_ordered_integration_test.go`
|
||||||
|
|
||||||
## Supplemental Validation Context
|
## Supplemental Validation Context
|
||||||
|
|
||||||
Request construction prepares one segment-position lookup and protected vocabulary
|
Each stage prepares a context matcher and indexes its selected glossary entries
|
||||||
for all candidates. Glossary selection remains candidate-specific, including the
|
by identity. Request construction uses a segment-position lookup and returns
|
||||||
proposed replacement text.
|
matched entries in selection order for each candidate, including matches in the
|
||||||
|
replacement text. The deterministic guard and application check share the
|
||||||
|
stage's separate protection matcher.
|
||||||
|
|
||||||
Each candidate includes the immediately preceding and following source segments
|
Each candidate includes the immediately preceding and following source segments
|
||||||
available within its fixed chunk, labeled `preceding` or `following` with speaker,
|
available within its fixed chunk, labeled `preceding` or `following` with speaker,
|
||||||
text, and categories, plus glossary entries whose names, aliases, or
|
text, and categories, plus glossary entries whose names, aliases, or
|
||||||
recognized plurals appear in its original/resulting text or those neighbors.
|
recognized plurals appear in its original/resulting text or those neighbors.
|
||||||
Glossary matching reuses the protected vocabulary; entries retain glossary order.
|
Glossary matching uses the same recognition rules; entries retain selection order.
|
||||||
Model-facing candidate and neighboring context omit internal segment IDs.
|
Model-facing candidate and neighboring context omit internal segment IDs.
|
||||||
This supplies evidence such as a troupe definition for a proposed `Jesters`
|
This supplies evidence such as a troupe definition for a proposed `Jesters`
|
||||||
correction or an adjacent speaker turn explaining deliberate repetition. A correction
|
correction or an adjacent speaker turn explaining deliberate repetition. A correction
|
||||||
@@ -140,7 +150,8 @@ payload token estimate. If one enriched item exceeds the configured limit, optio
|
|||||||
neighbors are removed from the end first, then glossary entries from the end,
|
neighbors are removed from the end first, then glossary entries from the end,
|
||||||
until it fits or only required candidate data remains. Required before/after text
|
until it fits or only required candidate data remains. Required before/after text
|
||||||
is never truncated. Existing oversized-candidate handling still applies if that
|
is never truncated. Existing oversized-candidate handling still applies if that
|
||||||
required data does not fit. Context is supplemental, not a promise that every
|
required data does not fit. The built-in estimator reuses encoded fragment
|
||||||
|
counts for fitting and batching. Context is supplemental, not a promise that every
|
||||||
relevant definition or neighboring utterance always fits. This uses the existing
|
relevant definition or neighboring utterance always fits. This uses the existing
|
||||||
payload estimator, not a new guarantee about the provider's full context window.
|
payload estimator, not a new guarantee about the provider's full context window.
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
||||||
Version-1 configuration files, environment overlays, and retired flags are not
|
Version-1 configuration files, environment overlays, and retired flags are not
|
||||||
accepted. Copy a maintained [local](../../examples/minimal-config.yml) or
|
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or
|
||||||
[service](../../examples/production-config.yml) example, then select real models
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/production-config.yml) example, then select real models
|
||||||
in the accompanying `examples/profiles/` files. A testing overlay is shown at
|
in the accompanying `examples/profiles/` files. A testing overlay is shown at
|
||||||
`examples/overlays/testing.yml`; its model choice carries no cost, speed, or
|
`examples/overlays/testing.yml`; its model choice carries no cost, speed, or
|
||||||
quality guarantee.
|
quality guarantee.
|
||||||
@@ -122,7 +122,7 @@ capacity and usage fields retain known/complete flags. Debug runs write
|
|||||||
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
|
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
|
||||||
`config-origins.yaml`, `utilization-diagnostics.json`, and
|
`config-origins.yaml`, `utilization-diagnostics.json`, and
|
||||||
`llm-execution.json`. Request and response artifacts remain redacted debugging
|
`llm-execution.json`. Request and response artifacts remain redacted debugging
|
||||||
material. See [operations](../operations.md#diagnostics-directory) for placement
|
material. See [operations](../operations.md#debug-evidence) for placement
|
||||||
and paths, [configuration](../config.md) for all settings, and the
|
and paths, [configuration](../config.md) for all settings, and the
|
||||||
[CLI reference](../cli.md) for current flags.
|
[CLI reference](../cli.md) for current flags.
|
||||||
|
|
||||||
|
|||||||
27
docs/migrations/v3.md
Normal file
27
docs/migrations/v3.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Migrating to application configuration v3
|
||||||
|
|
||||||
|
Audita 3.1.x uses application configuration schema v2. For the cutover to schema
|
||||||
|
v3, set `version: 3` in the application file and move glossary entries into
|
||||||
|
configuration. Use the [configuration reference](../config.md) and
|
||||||
|
[maintained examples](../../examples/) for the new formats.
|
||||||
|
|
||||||
|
- Replace `pipeline.modules` with ordered `pipeline.stages` records containing
|
||||||
|
`module` and any per-entry overrides. An explicit empty list runs normalization.
|
||||||
|
- Define named `glossary_collections` in configuration or its companion imports;
|
||||||
|
select them using `glossaries` and `protected_terms`. Rename each legacy
|
||||||
|
entry's `summary` to `description` to preserve its prompt context. Remove
|
||||||
|
`--glossary` and `--modules` from invocations.
|
||||||
|
- Move profiles and backends out of the application configuration and retired
|
||||||
|
`profiles_dir` layout into PromptKit resources. Use its current list records,
|
||||||
|
IDs, credential sources, and backend admission settings. Select proposal and
|
||||||
|
validation profiles in application `llm` settings.
|
||||||
|
- Use the shared `--debug` flag for capture; `diagnostics.work_dir` remains the
|
||||||
|
parent directory. Review report and output destinations before running.
|
||||||
|
- Completion counts are hidden by default. Use `--verbose` to see them on
|
||||||
|
stderr, including when ordinary reports are disabled. `--debug` also shows
|
||||||
|
counts unless followed by `--quiet`. See [CLI reporting](../cli.md#processing).
|
||||||
|
|
||||||
|
Inspect configuration and resources before processing, then try a controlled
|
||||||
|
small transcript. Processing preflight checks every glossary correction and
|
||||||
|
selected role before preparing outputs or invoking a model. See [operations](../operations.md)
|
||||||
|
for lifecycle and recovery.
|
||||||
@@ -1,94 +1,66 @@
|
|||||||
# Audita Operations
|
# Audita Operations
|
||||||
|
|
||||||
This guide owns physical output placement and failure handling. The
|
This guide owns output placement and failure handling. The [CLI reference](cli.md)
|
||||||
[CLI reference](cli.md) owns flags and streams, [configuration](config.md) owns
|
owns invocation, [configuration](config.md) owns settings, and the
|
||||||
settings, and the [output contract](integrations/output-files.md) owns schemas.
|
[output contract](integrations/output-files.md) owns serialized formats.
|
||||||
|
|
||||||
## Normal Workflow
|
## Processing
|
||||||
|
|
||||||
1. Validate a version-2 configuration and its selected profiles with
|
Inspect the application and PromptKit resources, prepare a transcript, then run
|
||||||
`audita config validate --config config.yml`.
|
`audita process` with the desired destination. The [examples](../examples/README.md)
|
||||||
2. Prepare a transcript and glossary following the [input contract](integrations/transcript-glossary-files.md).
|
show complete projects. Processing preflight checks every configured glossary
|
||||||
3. Run `audita process input.json --config config.yml --glossary glossary.yaml --output corrected.json`.
|
correction and selected profile before output preparation and model work. Empty
|
||||||
4. Check the exit status, stderr summary, corrected transcript, and reports.
|
protection and an empty pipeline are valid. A completed run can contain
|
||||||
A completed run can contain recoverable model errors and still apply
|
recoverable model errors while still applying independently validated edits.
|
||||||
corrections that passed every required validator.
|
Correction repair sends the complete original proposal exchange and branch
|
||||||
|
history to the selected repair provider. A provider rejection ends that branch
|
||||||
|
as unavailable; other approved edits can still be applied. See
|
||||||
|
[configuration](config.md#settings) for the attempt budget and profile selection.
|
||||||
|
|
||||||
## Required Output Files
|
## Outputs and lifecycle
|
||||||
|
|
||||||
Reporting is enabled by default. For file transcript output, `errors.yml`,
|
Reporting is enabled by default. For transcript file output, `errors.yml`,
|
||||||
`recoveries.yml`, and `report.md` go in the transcript output's parent directory
|
`recoveries.yml`, and `report.md` are placed beside that file unless `--report-dir`
|
||||||
unless `--report-dir` is explicitly supplied. For stdout transcript output, they
|
is supplied. For stdout transcript output they use the effective report directory.
|
||||||
go in the effective `reporting.stdout_dir` (default `.`), including a
|
Disabling reporting leaves its destination inactive and unprobed. Quiet mode
|
||||||
`--report-dir` override. A config-file or overlay value for
|
requires a transcript file or `/dev/null`; it suppresses optional messages, not
|
||||||
`reporting.stdout_dir` alone does not redirect file-output reports. An active
|
failures. Stdout carries transcript JSON when selected.
|
||||||
selected report directory must be nonblank. `--reporting=false` suppresses all
|
|
||||||
three reports and leaves their destination inactive, so it is not probed merely
|
|
||||||
because `--report-dir` was supplied. It does not suppress the stderr summary or
|
|
||||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
|
||||||
|
|
||||||
Existing report directories and parents for regular transcript files must be
|
Before processing, Audita checks destination collisions against the transcript,
|
||||||
writable. Audita prevents report writes from overwriting transcript, glossary,
|
consumed application and PromptKit sources, and effective host message files. An
|
||||||
loaded application configuration, host YAML discovered under `profiles_dir`,
|
intentional in-place transcript replacement is allowed. A regular transcript
|
||||||
or host message files referenced by effective PromptKit `content_file` fields,
|
output is prepared in a sibling temporary file, leaving the final destination
|
||||||
and rejects conflicting output destinations. Source paths come from the
|
untouched until successful publication. Pipes and devices are opened only for
|
||||||
runtime's captured configuration; unrelated files in the selected directory
|
delivery, and a pipe can wait for its reader. If a transcript destination changes
|
||||||
remain eligible output destinations. The final transcript may intentionally
|
between a regular file and a pipe or device during processing, Audita can report a
|
||||||
replace its own input after successful processing; other input/configuration/
|
delivery failure while preserving the new destination. Parent directories for
|
||||||
profile sources remain protected.
|
required outputs must already exist.
|
||||||
Preparation never truncates an existing transcript. Device and pipe destinations
|
|
||||||
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
|
||||||
they are not opened during preflight or replaced with regular files. A pipe write
|
|
||||||
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
|
||||||
reports still use the output path's parent unless an explicit `--report-dir`
|
|
||||||
selects another directory; disable reporting when that placement is unsuitable
|
|
||||||
for a stream destination.
|
|
||||||
|
|
||||||
With reporting enabled, all three reports are initialized as `in_progress` before
|
Enabled reports are initialized as `in_progress` before processing. The
|
||||||
input processing. Output is encoded fully before publication. Each regular output
|
transcript is published before final reports; the files do not form a single
|
||||||
file is replaced through a temporary file on its destination filesystem:
|
transaction. A late report failure may therefore leave a published transcript.
|
||||||
transcript first, then `errors.yml`,
|
Fatal errors and cancellation clean unpublished temporary resources and attempt
|
||||||
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
|
failed-status final reports. An uncatchable termination can leave `in_progress`
|
||||||
multi-file transaction. New files honor the process umask; replacing an existing
|
evidence. Read exit status and reports together rather than treating a file's
|
||||||
file preserves its permission bits. A late report error may leave the transcript
|
existence as proof of success.
|
||||||
published; the process exits nonzero and attempts failed-status reports. On fatal processing
|
|
||||||
or cancellation, no completed transcript is published, but partial reports are
|
|
||||||
finalized where possible. An uncatchable termination can leave `in_progress`.
|
|
||||||
|
|
||||||
## Diagnostics Directory
|
## Debug evidence
|
||||||
|
|
||||||
`--debug` or `diagnostics.debug` enables detailed evidence independently of
|
The shared `--debug` flag enables retained artifacts in a run directory beneath
|
||||||
reporting. One directory named `run-<id>` is created under
|
`diagnostics.work_dir`; selecting a work directory alone does not enable capture.
|
||||||
`diagnostics.work_dir` (default `/tmp/audita`), which `--work-dir` can override,
|
Optional debug preparation or writes can produce redacted warnings without
|
||||||
and retained on success or failure. Selecting `--work-dir` alone does not enable
|
changing correction decisions or the required output result. Debug material can
|
||||||
capture. Without debug, no directory or payload files are created. The parent
|
include transcript content, prompts, responses, and correction decisions. It is
|
||||||
is created when needed only for debug. Failure to
|
not anonymized; protect it according to the input's sensitivity. Configured
|
||||||
write optional debug evidence prints a redacted stderr diagnostic but does not
|
credential values are redacted at the diagnostic boundary. Repair interactions
|
||||||
change correction decisions or the exit status.
|
and revalidation artifacts include proposal and attempt coordinates; the
|
||||||
|
correction ledger connects them to the original edit.
|
||||||
|
|
||||||
Debug evidence includes invocation and effective settings, configuration
|
## Recovery
|
||||||
origins, selected profiles, redacted source and normalized snapshots,
|
|
||||||
normalization and initial chunk summaries, all captured proposed candidates
|
|
||||||
and validator decisions in `correction-ledger.json`, execution/utilization
|
|
||||||
records, and model interaction files. Individual artifact writes can fail; a
|
|
||||||
directory's existence does not prove every file was written. These files can
|
|
||||||
contain transcript text and prompt/response payloads. Configured backend
|
|
||||||
credentials are redacted, but the transcript itself is not anonymized. Protect
|
|
||||||
and retain the directory according to the sensitivity of the input.
|
|
||||||
|
|
||||||
## Failure Inspection and Recovery
|
Exit `0` indicates completion of processing and required outputs; nonzero status
|
||||||
|
indicates an input, processing, cancellation, or required-output failure.
|
||||||
Exit `0` means processing and every enabled required output completed. Exit
|
Recoverable model-operation failures and successful repairs or retries are
|
||||||
`1` means cancellation, fatal processing, or required-output failure. Exit `2`
|
recorded separately in the ordinary YAML reports. The Markdown matrices identify
|
||||||
identifies usage or configuration/destination selection errors. Recoverable
|
pipeline position, module, and optional operator ID. Fix the input, service, or
|
||||||
model-operation failures appear in `errors.yml` as one record per affected
|
destination and rerun when necessary; Audita has no resume or checkpoint API.
|
||||||
work unit; they do not automatically make the run fail. `recoveries.yml`
|
|
||||||
records successful operations that needed retry, repair, or proposal filtering.
|
|
||||||
`report.md` shows every initial chunk and configured module instance, including
|
|
||||||
interrupted and unattempted work. Ordinary reports omit transcript excerpts and
|
|
||||||
full decision ledgers; enable debug when those are needed.
|
|
||||||
|
|
||||||
Read stderr and report status first. If a required output failed after a
|
|
||||||
transcript was published, do not treat the file alone as proof of a completed
|
|
||||||
run. Fix the input, service, or destination problem and rerun. Audita has no
|
|
||||||
resume/checkpoint API.
|
|
||||||
|
|||||||
@@ -4,9 +4,10 @@
|
|||||||
|
|
||||||
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy
|
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy
|
||||||
Reading Guide in `docs/development.md`, adapted for this repository.
|
Reading Guide in `docs/development.md`, adapted for this repository.
|
||||||
`docs/policy/release.md` incorporates revision
|
`docs/policy/documentation.md`, `docs/policy/oss.md`, and `docs/policy/release.md`
|
||||||
`e833faddf120c789a9b45348edaf75720de16348`; the remaining listed material
|
incorporate revision `593c9b52273c5c879785ae9e995d21026df467b4`.
|
||||||
incorporates revision `d84a38ebab26f048cafbe59a2246700c35d29843`.
|
The remaining listed material incorporates revision
|
||||||
|
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -15,16 +16,16 @@ SPDX-License-Identifier: CC-BY-SA-4.0
|
|||||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4).
|
||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -32,13 +33,16 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT).
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e).
|
||||||
|
|
||||||
## ReleaseKit
|
## ReleaseKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ReleaseKit` at `v0.4.1`.
|
Selected module: `go.maximumdirect.net/releasekit` at `v0.9.0`.
|
||||||
|
|
||||||
|
Material: ReleaseKit runtime and adapted launcher and tag-delivery recipes in
|
||||||
|
`scripts/release.sh` and `.woodpecker/release.yml`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -46,79 +50,121 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ReleaseKit/src/commit/2d5a8110c61e31daf8698766114ae305d0e0e469/COPYRIGHT).
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ReleaseKit/src/commit/2d5a8110c61e31daf8698766114ae305d0e0e469).
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4).
|
||||||
|
|
||||||
|
## CommandKit
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/commandkit` at `v0.6.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/1613ff95cab60581de5d86f6e9e2a3cc091a44a0/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/1613ff95cab60581de5d86f6e9e2a3cc091a44a0).
|
||||||
|
|
||||||
|
## CommandKit ckconfig
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/ckconfig` at `v0.7.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/64f6453b8453f914f272d427bb4ae6f59deacd2b/integrations/ckconfig/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/64f6453b8453f914f272d427bb4ae6f59deacd2b).
|
||||||
|
|
||||||
|
## CommandKit ckprompt
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/ckprompt` at `v0.7.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/integrations/ckprompt/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0).
|
||||||
|
|
||||||
## Go runtime and standard library
|
## Go runtime and standard library
|
||||||
|
|
||||||
Selected toolchain: Go 1.26.8 for the distributed Linux binaries.
|
Selected toolchain: Go 1.27.3 for the distributed Linux binaries.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e).
|
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
|
||||||
|
|
||||||
## Go vendored golang.org/x/crypto
|
## Go vendored golang.org/x/crypto
|
||||||
|
|
||||||
Material: `golang.org/x/crypto` at `v0.46.1-0.20251210140736-7dacc380ba00` in the Go 1.26.8 standard library.
|
Material: `golang.org/x/crypto` at `v0.52.1-0.20260526024921-9beb694f9766` in the Go 1.27.3 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/crypto/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/crypto/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/crypto/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/crypto/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e).
|
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
|
||||||
|
|
||||||
## Go vendored golang.org/x/net
|
## Go vendored golang.org/x/net
|
||||||
|
|
||||||
Material: `golang.org/x/net` at `v0.47.1-0.20260731170545-cb1d721f7fb3` in the Go 1.26.8 standard library.
|
Material: `golang.org/x/net` at `v0.55.1-0.20260731170536-c1d18010be90` in the Go 1.27.3 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/net/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/net/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/net/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/net/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e).
|
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
|
||||||
|
|
||||||
## Go vendored golang.org/x/sys
|
## Go vendored golang.org/x/sys
|
||||||
|
|
||||||
Material: `golang.org/x/sys` at `v0.39.0` in the Go 1.26.8 standard library.
|
Material: `golang.org/x/sys` at `v0.45.0` in the Go 1.27.3 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/sys/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/sys/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/sys/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/sys/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e).
|
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
|
||||||
|
|
||||||
## Go vendored golang.org/x/text
|
## Go vendored golang.org/x/text
|
||||||
|
|
||||||
Material: `golang.org/x/text` at `v0.32.0` in the Go 1.26.8 standard library.
|
Material: `golang.org/x/text` at `v0.37.0` in the Go 1.27.3 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/text/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/text/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e/src/vendor/golang.org/x/text/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/text/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/c293dd49cbe25e1fe8d97d94a5cb618e7b6d831e).
|
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
|
||||||
|
|
||||||
## go-yaml v4
|
## go-yaml v4
|
||||||
|
|
||||||
@@ -164,7 +210,7 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
|
|||||||
|
|
||||||
## PromptKit
|
## PromptKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.14.0`.
|
Selected module: `go.maximumdirect.net/promptkit` at `v0.21.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -172,37 +218,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/143c6e9b3a675872117b0471e2ff8bcb25055a54/COPYRIGHT).
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/143c6e9b3a675872117b0471e2ff8bcb25055a54).
|
[Source](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
|
||||||
|
|
||||||
## yaml.v3
|
## yaml.v3
|
||||||
|
|
||||||
@@ -224,3 +242,31 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
|||||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||||
|
|
||||||
[Source](https://github.com/go-yaml/yaml/commit/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
[Source](https://github.com/go-yaml/yaml/commit/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||||
|
|
||||||
|
## Cobra
|
||||||
|
|
||||||
|
Selected module: `github.com/spf13/cobra` at `v1.10.1`.
|
||||||
|
|
||||||
|
Copyright 2013-2023 The Cobra Authors
|
||||||
|
|
||||||
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||||
|
[LICENSE](https://github.com/spf13/cobra/blob/7da941c3547e93b8c9f70bbd3befca79c6335388/LICENSE.txt).
|
||||||
|
|
||||||
|
[Source](https://github.com/spf13/cobra/tree/7da941c3547e93b8c9f70bbd3befca79c6335388).
|
||||||
|
|
||||||
|
## pflag
|
||||||
|
|
||||||
|
Selected module: `github.com/spf13/pflag` at `v1.0.9`.
|
||||||
|
|
||||||
|
Copyright (c) 2012 Alex Ogier. All rights reserved.
|
||||||
|
|
||||||
|
Copyright (c) 2012 The Go Authors. All rights reserved.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
|
[LICENSE](https://github.com/spf13/pflag/blob/10438578954bba2527fe5cae3684d4532b064bbe/LICENSE).
|
||||||
|
|
||||||
|
[Source](https://github.com/spf13/pflag/tree/10438578954bba2527fe5cae3684d4532b064bbe).
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
|
||||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
|
||||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
|
||||||
following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
|
||||||
portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
|
||||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
|
||||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
||||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
||||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies architecture](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
**Upstream Source:** [Standard Repository Policies architecture](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||||
|
|
||||||
## Purpose And Ownership
|
## Purpose And Ownership
|
||||||
@@ -72,13 +72,19 @@ hidden global state.
|
|||||||
|
|
||||||
## Correction and Concurrency Invariants
|
## Correction and Concurrency Invariants
|
||||||
|
|
||||||
|
- Module focus guides proposal generation; it is not an exclusive acceptance
|
||||||
|
boundary. Validators permit otherwise acceptable minor editorial cleanup even
|
||||||
|
when the originating module did not request it. Existing applicability,
|
||||||
|
protection, and meaning checks still apply. See
|
||||||
|
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md) for the empirical
|
||||||
|
rationale and rejected strict-scope alternative.
|
||||||
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
||||||
indexing, validator ordering, transcript mutation, reports, and serialization.
|
indexing, validator ordering, transcript mutation, reports, and serialization.
|
||||||
- Validate corrections before final mutation, including local proposal checks,
|
- Validate corrections before final mutation, including local proposal checks,
|
||||||
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
||||||
does not replace these checks.
|
does not replace these checks.
|
||||||
- Keep LLM concurrency bounded and context-aware through PromptKit's engine-local
|
- Use PromptKit's context-aware, engine-local per-backend admission for
|
||||||
per-backend admission. Proposal and validation calls selecting the same backend
|
LLM concurrency. Proposal and validation calls selecting the same backend
|
||||||
share that capacity, including retries and repairs. Preserve deterministic
|
share that capacity, including retries and repairs. Preserve deterministic
|
||||||
collection order. Recoverable model-operation failures stay local to their
|
collection order. Recoverable model-operation failures stay local to their
|
||||||
proposal response or validation batch; fatal invariant/application failure
|
proposal response or validation batch; fatal invariant/application failure
|
||||||
@@ -120,13 +126,19 @@ responsibilities and should remain separate. Implementation details belong in
|
|||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
|
Keep Audita dependency-light and prefer the Go standard library where no adopted
|
||||||
filesystems, synchronization, and tests. ConfigKit owns configuration loading,
|
library owns the boundary. ConfigKit owns configuration composition, presence,
|
||||||
presence, provenance, and secret resolution; PromptKit owns prompt/profile
|
recorded resolution, and provenance. PromptKit owns prompt, profile, backend,
|
||||||
loading, OpenAI-compatible transport, retries, repairs, and backend admission.
|
credential, transport, retry, repair, and admission behavior. CommandKit and its
|
||||||
Audita owns domain meaning, role selection, and deterministic processing.
|
integrations own command routing and their standard inspection and request
|
||||||
|
surfaces. Audita owns domain meaning, role selection, transcript processing,
|
||||||
|
and presentation of its processing results.
|
||||||
|
|
||||||
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
|
Pass resolved typed configuration to preflight and processing. Preflight checks
|
||||||
|
execution readiness before output preparation or model work; the output session
|
||||||
|
owns resources, cleanup, and publication. The glossary and readiness decision is
|
||||||
|
recorded in [ADR 0002](../adr/0002-glossary-configuration-and-preflight.md).
|
||||||
|
Do not introduce broad dependency-injection, workflow, logging, or plugin
|
||||||
frameworks without a concrete implemented need and focused tests.
|
frameworks without a concrete implemented need and focused tests.
|
||||||
|
|
||||||
## Testing and Documentation
|
## Testing and Documentation
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Documentation Policy
|
# Documentation Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.11
|
**Base Policy Version:** 1.12
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md)
|
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/documentation.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -85,8 +85,8 @@ secret values.
|
|||||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
||||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
| Architectural decision records | `docs/adr/` | Records of significant architectural decisions and supporting rationale. | Ordinary design choices, current behavior reference, implementation status, task sequencing. |
|
||||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, or future work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
|
|
||||||
@@ -154,8 +154,15 @@ Use sequentially numbered ADR filenames such as
|
|||||||
|
|
||||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
|
||||||
belong in the roadmap.
|
ADRs should be reserved for major architectural decisions, such as ownership
|
||||||
|
boundaries, large-scale architectural features, or invariants that rise to
|
||||||
|
the level of architectural significance. Routine choices, implementation
|
||||||
|
decisions, and rejected suggestions do not warrant an ADR unless they
|
||||||
|
otherwise represent a significant architectural decision.
|
||||||
|
|
||||||
|
Roadmaps should preserve plausible future work, not rejected product
|
||||||
|
ideas.
|
||||||
|
|
||||||
## Maintenance
|
## Maintenance
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Open Source Licensing And Attribution Policy
|
# Open Source Licensing And Attribution Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.8
|
**Base Policy Version:** 1.9
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/oss.md)
|
||||||
|
|
||||||
## Purpose And Scope
|
## Purpose And Scope
|
||||||
|
|
||||||
@@ -64,9 +64,12 @@ license unless the author deliberately changes it. For mixed-purpose projects,
|
|||||||
follow the author's selected scope rather than inferring different licenses for
|
follow the author's selected scope rather than inferring different licenses for
|
||||||
individual packages.
|
individual packages.
|
||||||
|
|
||||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/application/COPYRIGHT)
|
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/library/COPYRIGHT)
|
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||||
to create the project’s root `COPYRIGHT` file.
|
to create the project’s root `COPYRIGHT` file.
|
||||||
|
Use the COPYRIGHT examples exactly as provided, except for placeholder
|
||||||
|
substitutions and changes expressly required or authorized by this policy or
|
||||||
|
the author.
|
||||||
Use verified project names, contribution years, and copyright holders.
|
Use verified project names, contribution years, and copyright holders.
|
||||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||||
text references together to reflect the author's selected grant.
|
text references together to reflect the author's selected grant.
|
||||||
@@ -113,7 +116,7 @@ Each entry presents these five fields in order, with blank lines between them:
|
|||||||
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
||||||
5. **Source repository link**, rendered exactly as `[Source](COMMIT_URL).`.
|
5. **Source repository link**, rendered exactly as `[Source](COMMIT_URL).`.
|
||||||
|
|
||||||
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/attribution.md)
|
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/attribution.md)
|
||||||
to create the project’s `docs/oss/attribution.md`.
|
to create the project’s `docs/oss/attribution.md`.
|
||||||
The example illustrates omitted fields and multiple copyrights and licenses.
|
The example illustrates omitted fields and multiple copyrights and licenses.
|
||||||
Replace the uppercase placeholders and illustrative URLs with actual component
|
Replace the uppercase placeholders and illustrative URLs with actual component
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Release Policy
|
# Release Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.6
|
**Base Policy Version:** 1.7
|
||||||
|
|
||||||
**Last Updated**: 2026-10-08
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/release.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -82,6 +82,14 @@ For releases at or after `v1.0.0`:
|
|||||||
minimum as described below; and
|
minimum as described below; and
|
||||||
- a patch release may make backward-compatible fixes.
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
|
In this context, "backward-compatible" means only that pre-existing, valid
|
||||||
|
configurations will continue to work following the release. Behavior need not
|
||||||
|
remain strictly identical.
|
||||||
|
|
||||||
|
Audita's minor and patch releases also preserve the documented
|
||||||
|
[subprocess integration](../integrations/subprocess.md) and
|
||||||
|
[output-file](../integrations/output-files.md) contracts.
|
||||||
|
|
||||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
configuration, data, integration, or compatibility contract when the release
|
configuration, data, integration, or compatibility contract when the release
|
||||||
note explains the impact and required consumer action. A patch release must
|
note explains the impact and required consumer action. A patch release must
|
||||||
@@ -184,6 +192,15 @@ only through an explicit project-specific policy decision explaining the
|
|||||||
concrete risk and why ordinary change review or automation is insufficient.
|
concrete risk and why ordinary change review or automation is insufficient.
|
||||||
Do not add routine confirmations for checks the command can perform itself.
|
Do not add routine confirmations for checks the command can perform itself.
|
||||||
|
|
||||||
|
## Audita's Selected Candidate Checks
|
||||||
|
|
||||||
|
Audita uses ReleaseKit v0.9.0 with race-enabled offline tests, full Go vet,
|
||||||
|
formatting, Markdown links, module distribution checks, and the required target
|
||||||
|
builds. Race checks address shared processing and client state; vet and format
|
||||||
|
checks catch common defects and mechanical drift; link and module checks protect
|
||||||
|
the distributed source tree. The [runbook](../release.md) specifies the selected
|
||||||
|
toolchain, native commands, and asynchronous recovery.
|
||||||
|
|
||||||
## Publication Guards
|
## Publication Guards
|
||||||
|
|
||||||
Automate the checks needed to publish the intended source under a fresh version:
|
Automate the checks needed to publish the intended source under a fresh version:
|
||||||
|
|||||||
@@ -7,21 +7,21 @@ the release; Woodpecker reports binary delivery separately.
|
|||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
Use Bash and Git. The authoritative remote is `origin`,
|
Use Bash, Git, a C compiler, and cgo for race detection. The authoritative
|
||||||
which must point to `https://gitea.maximumdirect.net/eric/audita.git`; the
|
remote is `origin`, which must point to `https://gitea.maximumdirect.net/eric/audita.git`; the
|
||||||
primary branch is `main`. Source publication needs no delivery credential.
|
primary branch is `main`. Source publication needs no delivery credential.
|
||||||
Woodpecker uses the existing `GITEA_RELEASE_TOKEN` secret for delivery.
|
Woodpecker uses the existing `GITEA_RELEASE_TOKEN` secret for delivery.
|
||||||
|
|
||||||
The source-build minimum is Go 1.26.0, declared in [go.mod](../go.mod).
|
The source-build minimum is Go 1.27.0, declared in [go.mod](../go.mod).
|
||||||
This minimum is required by the selected ReleaseKit dependency.
|
This minimum is required by the selected application dependencies.
|
||||||
Distributed executables do not require a Go installation. The selected release
|
Distributed executables do not require a Go installation. The selected release
|
||||||
toolchain is Go 1.26.8, matching the current Woodpecker image and distributed
|
toolchain is Go 1.27.3, matching the current Woodpecker image and distributed
|
||||||
runtime notices. Use that local installation for tool bootstrap, candidate
|
runtime notices. Use that local installation for tool bootstrap, candidate
|
||||||
validation, and release builds. Replace the installation path below with its
|
validation, and release builds. Replace the installation path below with its
|
||||||
actual directory:
|
actual directory:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
export PATH="/path/to/go1.26.8/bin:$PATH"
|
export PATH="/path/to/go1.27.3/bin:$PATH"
|
||||||
export GOTOOLCHAIN=local
|
export GOTOOLCHAIN=local
|
||||||
go version
|
go version
|
||||||
```
|
```
|
||||||
@@ -42,8 +42,9 @@ GOWORK=off GOTOOLCHAIN=local go mod download
|
|||||||
If dependency preparation changes `go.mod` or `go.sum`, include those changes in
|
If dependency preparation changes `go.mod` or `go.sum`, include those changes in
|
||||||
the committed candidate. ReleaseKit's CLI is registered as a Go tool dependency;
|
the committed candidate. ReleaseKit's CLI is registered as a Go tool dependency;
|
||||||
`go mod tidy` retains its dependencies and checksums. The release wrapper changes
|
`go mod tidy` retains its dependencies and checksums. The release wrapper changes
|
||||||
to the repository root and invokes `go tool releasekit` with workspace discovery
|
to the repository root, selects the host target, and invokes the full
|
||||||
and automatic toolchain downloads disabled. Go builds or reuses the selected
|
`go.maximumdirect.net/releasekit/cmd/releasekit` tool path with workspace
|
||||||
|
discovery and automatic toolchain downloads disabled. Go builds or reuses the selected
|
||||||
tool from its normal build cache. Tool preparation precedes offline candidate
|
tool from its normal build cache. Tool preparation precedes offline candidate
|
||||||
validation.
|
validation.
|
||||||
|
|
||||||
@@ -78,9 +79,11 @@ Publish from the repository root, substituting the selected version:
|
|||||||
bash scripts/release.sh publish VERSION
|
bash scripts/release.sh publish VERSION
|
||||||
```
|
```
|
||||||
|
|
||||||
The command runs `go test -mod=readonly -count=1 ./...` once with module and
|
The command runs the offline Go test suite with race detection, full Go vet,
|
||||||
toolchain network access disabled, builds the profile's Linux amd64 and arm64
|
formatting, Markdown link, and module distribution checks. Race detection covers
|
||||||
targets, and checks the native version output. It creates an annotated tag whose
|
shared processing and client state; vet and formatting catch common Go defects and
|
||||||
|
mechanical drift; the link and module checks protect the distributed source tree. It builds the
|
||||||
|
profile's Linux amd64 and arm64 targets and checks the native version output. It creates an annotated tag whose
|
||||||
message comes from the committed note, then pushes the exact tag object. A
|
message comes from the committed note, then pushes the exact tag object. A
|
||||||
successful publication completes the source release without waiting for CI or
|
successful publication completes the source release without waiting for CI or
|
||||||
checking artifact availability.
|
checking artifact availability.
|
||||||
@@ -104,11 +107,13 @@ Woodpecker runs ReleaseKit's `ci-deliver` command for tags under
|
|||||||
- `SHA256SUMS`
|
- `SHA256SUMS`
|
||||||
|
|
||||||
Before running Go, the workflow fetches and verifies the event tag, checks it
|
Before running Go, the workflow fetches and verifies the event tag, checks it
|
||||||
out detached, and confirms the source tree is clean. It copies the tagged
|
out detached, and confirms the source tree is clean. It peels the event object
|
||||||
|
ID to its commit before comparing it with the tag's commit, accepting both
|
||||||
|
annotated tag objects and commit IDs from Woodpecker. It copies the tagged
|
||||||
`go.mod` and `go.sum` into temporary module metadata outside the checkout. Go
|
`go.mod` and `go.sum` into temporary module metadata outside the checkout. Go
|
||||||
confirms that this configuration selects ReleaseKit, then builds its CLI using
|
confirms that this configuration selects ReleaseKit, then invokes its CLI
|
||||||
an alternate `-modfile`. The bootstrap executable and temporary files remain
|
using an alternate `-modfile` and the host target. Temporary metadata remains
|
||||||
outside the tagged source. The workflow runs with the selected Go 1.26.8 image,
|
outside the tagged source. The workflow runs with the selected Go 1.27.3 image,
|
||||||
`GOWORK=off`, and `GOTOOLCHAIN=local`.
|
`GOWORK=off`, and `GOTOOLCHAIN=local`.
|
||||||
|
|
||||||
Delivery uses prepared-checkout mode to verify the existing tag, HEAD, and clean
|
Delivery uses prepared-checkout mode to verify the existing tag, HEAD, and clean
|
||||||
@@ -117,10 +122,10 @@ worktree without repeating the bootstrap's fetch and checkout.
|
|||||||
The resulting executable receives:
|
The resulting executable receives:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
"$${bootstrap_dir}/releasekit" ci-deliver \
|
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
|
||||||
--repo "$${CI_WORKSPACE}" \
|
--repo "$${CI_WORKSPACE}" \
|
||||||
--prepared \
|
--prepared \
|
||||||
--commit "$${CI_COMMIT_SHA}" \
|
--commit "$${expected_commit}" \
|
||||||
"$${CI_COMMIT_TAG}"
|
"$${CI_COMMIT_TAG}"
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -131,8 +136,8 @@ after source publication; its status and logs report delivery progress or
|
|||||||
failure.
|
failure.
|
||||||
|
|
||||||
ReleaseKit handles content-aware retries and retained-artifact recovery. See
|
ReleaseKit handles content-aware retries and retained-artifact recovery. See
|
||||||
the pinned [publication recovery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/publish.md)
|
the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md)
|
||||||
and [delivery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/delivery.md)
|
and [delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md)
|
||||||
for those mechanics. A delivery failure does not undo a confirmed source
|
for those mechanics. A delivery failure does not undo a confirmed source
|
||||||
release. Retry the same immutable tag after resolving the reported state; do not
|
release. Retry the same immutable tag after resolving the reported state; do not
|
||||||
replace published files or a checksum manifest.
|
replace published files or a checksum manifest.
|
||||||
@@ -145,11 +150,11 @@ captured authoritative endpoint and publication state. Resolve the result
|
|||||||
against that endpoint before retrying. An exact remote tag-object match, with
|
against that endpoint before retrying. An exact remote tag-object match, with
|
||||||
its source available on remote `main`, can confirm publication. If the remote
|
its source available on remote `main`, can confirm publication. If the remote
|
||||||
state cannot be read, the result remains unknown; do not remove or replace the
|
state cannot be read, the result remains unknown; do not remove or replace the
|
||||||
tag while uncertain. Follow the pinned [publication recovery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/publish.md)
|
tag while uncertain. Follow the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md)
|
||||||
for object-level recovery details.
|
for object-level recovery details.
|
||||||
|
|
||||||
A source-release success and artifact-delivery status are separate. For a
|
A source-release success and artifact-delivery status are separate. For a
|
||||||
partial or uncertain delivery, inspect the job output and follow the pinned
|
partial or uncertain delivery, inspect the job output and follow the pinned
|
||||||
[delivery recovery guidance](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/delivery.md).
|
[delivery recovery guidance](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md).
|
||||||
A source or published-content correction requires a new appropriate version;
|
A source or published-content correction requires a new appropriate version;
|
||||||
leave the original tag and its contents intact.
|
leave the original tag and its contents intact.
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
# Release Notes
|
# Release Notes
|
||||||
|
|
||||||
|
- [Audita v3.1.1](v3.1.1.md)
|
||||||
- [Audita v3.1.0](v3.1.0.md)
|
- [Audita v3.1.0](v3.1.0.md)
|
||||||
- [Audita v3.0.1](v3.0.1.md)
|
- [Audita v3.0.1](v3.0.1.md)
|
||||||
- [Audita v3.0.0](v3.0.0.md)
|
- [Audita v3.0.0](v3.0.0.md)
|
||||||
|
|||||||
@@ -38,8 +38,8 @@ profile files require migration from the v2 release format.
|
|||||||
3. Select a configuration root whose final definitions are valid. PromptKit now
|
3. Select a configuration root whose final definitions are valid. PromptKit now
|
||||||
validates unused definitions too, including backend references, and conflicting
|
validates unused definitions too, including backend references, and conflicting
|
||||||
values within a directory source group are errors. The maintained examples
|
values within a directory source group are errors. The maintained examples
|
||||||
use separate [local](../../examples/profiles/local/) and
|
use separate [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/local/) and
|
||||||
[service](../../examples/profiles/service/) roots. Runtime source edits take
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/service/) roots. Runtime source edits take
|
||||||
effect on the next invocation.
|
effect on the next invocation.
|
||||||
4. Rename any application backend literally named `extends` and update its
|
4. Rename any application backend literally named `extends` and update its
|
||||||
profile references. ConfigKit now reserves that key for inheritance.
|
profile references. ConfigKit now reserves that key for inheritance.
|
||||||
@@ -47,11 +47,11 @@ profile files require migration from the v2 release format.
|
|||||||
`audita config validate --config /path/to/config.yml`, including any selected
|
`audita config validate --config /path/to/config.yml`, including any selected
|
||||||
testing overlay. Application configuration continues to use `version: 2`.
|
testing overlay. Application configuration continues to use `version: 2`.
|
||||||
|
|
||||||
The [configuration reference](../config.md#promptkit-profiles-and-operator-prompts)
|
The [v3.0.0 configuration reference](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/config.md#promptkit-profiles-and-operator-prompts)
|
||||||
contains the migrated format and presence rules. Start from the maintained
|
contains the migrated format and presence rules. Start from the
|
||||||
[local](../../examples/minimal-config.yml),
|
[local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/minimal-config.yml),
|
||||||
[service](../../examples/production-config.yml), or
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/production-config.yml), or
|
||||||
[application inheritance](../../examples/configkit-inheritance.yml) examples.
|
[application inheritance](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/configkit-inheritance.yml) examples.
|
||||||
The [CLI reference](../cli.md) describes report selection, and the
|
The [CLI reference](../cli.md) describes report selection, and the
|
||||||
[release procedure](../release.md) describes source publication and asynchronous
|
[release procedure](../release.md) describes source publication and asynchronous
|
||||||
Linux binary delivery.
|
Linux binary delivery.
|
||||||
|
|||||||
20
docs/releases/v3.1.1.md
Normal file
20
docs/releases/v3.1.1.md
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# Audita v3.1.1
|
||||||
|
|
||||||
|
This patch fixes the Woodpecker bootstrap failure that prevented Linux binary
|
||||||
|
delivery for v3.1.0. Audita's transcript processing and configuration are unchanged.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Resolve Woodpecker's tag-event object ID to its source commit before checking
|
||||||
|
the tagged checkout and invoking ReleaseKit. Annotated tag events previously
|
||||||
|
failed a comparison between the tag object and commit, stopping the workflow
|
||||||
|
before binary builds began.
|
||||||
|
- Add offline regression coverage for annotated and lightweight tag events,
|
||||||
|
including rejection of an unrelated source commit.
|
||||||
|
|
||||||
|
No configuration, profile, transcript-format, or command-behavior migration is
|
||||||
|
required from v3.1.0. The Go source-build minimum remains 1.26.0. Linux amd64 and
|
||||||
|
arm64 binary delivery follows the existing asynchronous
|
||||||
|
[release procedure](../release.md#binary-delivery).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/docs/oss/attribution.md)
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Deferred Feature Work
|
# Deferred Feature Work
|
||||||
|
|
||||||
**Last updated:** 2026-10-11
|
**Last updated:** 2026-11-11
|
||||||
|
|
||||||
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
||||||
entries preserve work beyond completed feature and library upgrades.
|
entries preserve work beyond completed feature and library upgrades.
|
||||||
@@ -10,10 +10,11 @@ implementing every capability.
|
|||||||
## Domain-Aware Response Recovery
|
## Domain-Aware Response Recovery
|
||||||
|
|
||||||
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
|
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
|
||||||
preserves usable decisions without another model call. Guided recovery remains
|
preserves usable decisions without another model call. Revision of rejected
|
||||||
deferred pending evidence from failures that remain. General per-entry
|
correction proposals is implemented with substantive
|
||||||
salvage from structurally invalid responses is also a separate design question;
|
validator feedback and explicit proposal withdrawal; see the
|
||||||
the accepted reconciliation feature retains whole-response structural validation.
|
[pipeline guide](../internal/pipeline.md).
|
||||||
|
It is distinct from recovering an unavailable validator decision.
|
||||||
|
|
||||||
Consider bounded feedback and regeneration for model responses that satisfy the
|
Consider bounded feedback and regeneration for model responses that satisfy the
|
||||||
structural schema but leave requested decisions unavailable after reconciliation.
|
structural schema but leave requested decisions unavailable after reconciliation.
|
||||||
@@ -30,11 +31,15 @@ secrets, and how regeneration fits bounded budgets and operation deadlines. Reus
|
|||||||
shared library capabilities where appropriate; do not introduce an uncoordinated
|
shared library capabilities where appropriate; do not introduce an uncoordinated
|
||||||
outer retry loop that multiplies existing retry/repair budgets.
|
outer retry loop that multiplies existing retry/repair budgets.
|
||||||
|
|
||||||
Ordinary editorial rejection is a valid decision, not a reason to regenerate until
|
Editorial rejection remains a valid decision about the evaluated proposal. The
|
||||||
an edit is approved. Recovery must preserve required validation, candidate identity,
|
implemented correction repair loop handles subsequent revision and full
|
||||||
partial-success safety, and truthful error/recovery reporting. Validate with
|
revalidation; this deferred entry concerns unavailable decisions. Decision recovery
|
||||||
controlled responses that distinguish recoverable protocol defects from valid
|
must preserve required validation, candidate identity, partial-success safety,
|
||||||
negative decisions. Exact APIs, budgets, and sequencing remain undecided.
|
and truthful error/recovery reporting. Validate with controlled responses that
|
||||||
|
distinguish protocol defects from valid negative decisions. Exact APIs, budgets,
|
||||||
|
and sequencing remain undecided. General per-entry salvage from structurally
|
||||||
|
invalid responses also remains a separate design question; current reconciliation
|
||||||
|
retains whole-response structural validation.
|
||||||
|
|
||||||
## Independent Whole-Pipeline Traversal by Chunk
|
## Independent Whole-Pipeline Traversal by Chunk
|
||||||
|
|
||||||
@@ -78,19 +83,3 @@ semantics. It must address meaning preservation and the effects on segment
|
|||||||
identity, timestamps, fixed chunks, subsequent stages, recombination, and output
|
identity, timestamps, fixed chunks, subsequent stages, recombination, and output
|
||||||
consumers. Do not infer authorization to remove a segment from an empty string
|
consumers. Do not infer authorization to remove a segment from an empty string
|
||||||
or introduce segment-removal machinery as part of span-deletion consistency.
|
or introduce segment-removal machinery as part of span-deletion consistency.
|
||||||
|
|
||||||
## Library Inspection and Configuration Comparison
|
|
||||||
|
|
||||||
ConfigKit and PromptKit provide effective-configuration comparison; PromptKit
|
|
||||||
also provides prompt/profile listings and contextual prompt requirement
|
|
||||||
inspection. Exposing these capabilities in Audita is deferred until a concrete
|
|
||||||
operator or development need arises. The dependency upgrade did not select new
|
|
||||||
inspection commands.
|
|
||||||
|
|
||||||
If a need emerges, define command ownership, selection syntax, report scope, and
|
|
||||||
disclosure before implementation. Reuse the libraries' public APIs and consider
|
|
||||||
shared CLI conventions alongside any separately selected CommandKit adoption.
|
|
||||||
Requirement discovery is advisory evidence rather than proof of execution
|
|
||||||
readiness; an empty configuration diff does not prove equivalent execution.
|
|
||||||
The library-upgrade review remains available in
|
|
||||||
[Git history](https://gitea.maximumdirect.net/eric/audita/src/commit/4c5536ab78c72dea58d03b3c51834138d429a59a/docs/roadmap/imports.md).
|
|
||||||
|
|||||||
103
docs/roadmap/prompt-quality.md
Normal file
103
docs/roadmap/prompt-quality.md
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
# Prompt Quality Roadmap
|
||||||
|
|
||||||
|
**Updated:** 2026-11-11
|
||||||
|
|
||||||
|
**Status:** Topic 1 resolved in ADR 0003; topics 2–6 remain for discussion.
|
||||||
|
Work through the remaining topics in order. Their recommendations are starting
|
||||||
|
points for discussion, not newly adopted restrictions or guarantees.
|
||||||
|
|
||||||
|
## Baseline and Intended Direction
|
||||||
|
|
||||||
|
The review of revision `6d34602` covered all eleven
|
||||||
|
[prompt definitions](../internal/prompts.md), their shared instructions, and
|
||||||
|
the context and response handling for proposals, validation, and repair.
|
||||||
|
It was a design review, not a measurement of live-model quality.
|
||||||
|
|
||||||
|
Retain the strengths: conservative transcript editing, explicit literal-span and
|
||||||
|
deletion semantics, distinct glossary context and protection, full before/after
|
||||||
|
validation, independent repair conversations with withdrawal, and native
|
||||||
|
PromptKit/ConfigKit prompt replacement. Improve consistency across the three
|
||||||
|
roles, useful rejection feedback, and maintainability.
|
||||||
|
|
||||||
|
## Discussion and Work Order
|
||||||
|
|
||||||
|
### 1. Module Focus and Editorial Acceptance — Resolved
|
||||||
|
|
||||||
|
The intentional distinction between requested module focus and acceptance of
|
||||||
|
minor editorial cleanup is recorded in
|
||||||
|
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md). The maintainer's
|
||||||
|
testing found that strict scope instructions also suppress desirable edits.
|
||||||
|
Apply the accepted distinction throughout the remaining prompt work.
|
||||||
|
|
||||||
|
### 2. Make Rejection Reasons Useful for Repair
|
||||||
|
|
||||||
|
Validator explanations now become repair feedback, but current instructions give
|
||||||
|
little guidance about producing actionable reasons.
|
||||||
|
|
||||||
|
**Recommendation:** ask for a concise description of the specific change, the
|
||||||
|
applicable criterion, and relevant evidence, within the validator's remit.
|
||||||
|
For example: removing "might" turns a tentative statement into a definite one;
|
||||||
|
preserve the speaker's uncertainty. Start with the existing `reason` field.
|
||||||
|
Discuss examples and sufficient detail before considering schema expansion.
|
||||||
|
|
||||||
|
### 3. Make Repair a Reconsideration of the Edit's Merits
|
||||||
|
|
||||||
|
Withdrawal is already explicit. Clarify when the model should use it and how it
|
||||||
|
should respond when the original evidence remains insufficient.
|
||||||
|
|
||||||
|
**Recommendation:** ask the model to reassess the correction from the supplied
|
||||||
|
evidence, retain the module's focus and the accepted editorial allowance, and
|
||||||
|
withdraw unsupported proposals. Address the risk that numerical confidence
|
||||||
|
feedback merely encourages raising the score.
|
||||||
|
Do not automatically prohibit confidence-only revisions: legitimate reassessment
|
||||||
|
is possible. Discuss and evaluate that behavior before adding a business rule.
|
||||||
|
Retain the agreed full-history conversation model and existing repair lifecycle.
|
||||||
|
|
||||||
|
### 4. Simplify Instructions and Improve Examples
|
||||||
|
|
||||||
|
Restrictions and examples recur across proposal, validation, and repair assets;
|
||||||
|
the grammar prompt repeats its word-substitution boundary several ways.
|
||||||
|
|
||||||
|
**Recommendation:** distinguish shared editing principles, requested module focus,
|
||||||
|
validator acceptance criteria, and task-specific response contracts. Use small
|
||||||
|
shared fragments where they reduce drift. Replace some repetitive example lists
|
||||||
|
with contextual contrasts, such as
|
||||||
|
accidental repetition versus deliberate emphasis. Preserve useful distinctions;
|
||||||
|
shorter prompts alone are not the objective. Discuss what belongs in shared
|
||||||
|
fragments and what remains local to each definition.
|
||||||
|
|
||||||
|
### 5. Evaluate Editorial Quality
|
||||||
|
|
||||||
|
Rendering, schema, and controlled-response tests establish integration behavior,
|
||||||
|
but do not establish the quality of model editorial judgments.
|
||||||
|
|
||||||
|
**Recommendation:** assemble a modest evaluation set containing both needed edits
|
||||||
|
and text that should remain unchanged. Include ambiguous glossary matches,
|
||||||
|
intentional repetition, meaningful uses of apparent filler, punctuation that
|
||||||
|
changes meaning, and repairs that should be withdrawn. Compare unnecessary edits,
|
||||||
|
missed corrections, validator disagreement, and repair outcomes using independent
|
||||||
|
editorial assessment; validator approval alone is insufficient evidence.
|
||||||
|
|
||||||
|
Capture candidate examples during topics 1–4 and establish a baseline before
|
||||||
|
changing prompts. Discuss representative models, expected outcomes, and how to
|
||||||
|
run evaluations. Keep model-dependent evaluation separate from the default
|
||||||
|
offline test suite; no numerical quality gate is selected here.
|
||||||
|
|
||||||
|
### 6. Revisit Broader Product Choices
|
||||||
|
|
||||||
|
Use the preceding decisions and evaluation evidence to discuss these in order:
|
||||||
|
|
||||||
|
1. **Grammar naming and scope:** the current module is close to copy editing,
|
||||||
|
with an isolated article exception. Decide whether its name and scope convey
|
||||||
|
the intended requested work, preserving the accepted editorial allowance.
|
||||||
|
2. **Glossary/homophone overlap:** assess the benefit of their overlapping remit
|
||||||
|
and repeated glossary passes before changing the default sequence.
|
||||||
|
3. **Language and examples:** decide whether spoken English and campaign-specific
|
||||||
|
examples are the right bundled defaults, and how broadly those defaults
|
||||||
|
should apply.
|
||||||
|
4. **Meaning-reversal policy:** the guard can also exclude a genuine correction
|
||||||
|
of a mistranscription. Initially retain the conservative default while
|
||||||
|
explicitly evaluating that tradeoff.
|
||||||
|
|
||||||
|
These are product discussions, not selected renames, broader editing permissions,
|
||||||
|
language-support promises, or changes to validation policy.
|
||||||
82
examples/README.md
Normal file
82
examples/README.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# Maintained examples
|
||||||
|
|
||||||
|
Build Audita from the repository root with Go 1.27 or newer. The commands below
|
||||||
|
assume the executable is on `PATH`. Each project selects its main file from its
|
||||||
|
own directory, which also determines companion resources and overlay context.
|
||||||
|
All models, credentials, names, and endpoints are illustrative.
|
||||||
|
|
||||||
|
## Local processing
|
||||||
|
|
||||||
|
From `examples/local`, edit `prompts.yml` to match a reachable OpenAI-compatible
|
||||||
|
local service and its model. Then run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config config.yml
|
||||||
|
audita process --config config.yml --output corrected.json transcript.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Processing makes model requests. Reports are written beside the selected output
|
||||||
|
unless disabled. For an offline validator preview, from the same directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prepared audita/validators.meaning_reversal_review \
|
||||||
|
--config config.yml --profile local \
|
||||||
|
--var "PayloadJSON=$(cat validation-candidate.json)" \
|
||||||
|
--var 'TranscriptDescriptionBlock='
|
||||||
|
```
|
||||||
|
|
||||||
|
To preview the repair instruction and its output schema without inference:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prepared audita/modules.grammar.repair \
|
||||||
|
--config config.yml --profile local \
|
||||||
|
--var 'RepairRequestJSON={"proposal_ref":1,"correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.9},"findings":[]}'
|
||||||
|
```
|
||||||
|
|
||||||
|
These previews render supplied data without inference. The native command help
|
||||||
|
and [CLI reference](../docs/cli.md) describe additional inspection and validation.
|
||||||
|
|
||||||
|
## Service and overlay
|
||||||
|
|
||||||
|
From `examples/service`, edit the synthetic endpoint and models in `prompts.yml`.
|
||||||
|
Set `AUDITA_EXAMPLE_KEY` to your service credential before a real request. Inspect
|
||||||
|
without a provider call:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config config.yml --overlay testing
|
||||||
|
audita promptkit show profile service-test --config config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The example does not promise a provider's model availability, price, or quality.
|
||||||
|
|
||||||
|
## Edit a prompt and run again
|
||||||
|
|
||||||
|
From `examples/prompt-edit`, inspect the operator replacement:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prompt audita/modules.grammar.proposal --config config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit its message in `prompts.yml`, repeat the inspection, and run `audita process
|
||||||
|
--config config.yml --output corrected.json transcript.json` against a configured
|
||||||
|
local service. Each invocation captures the files anew.
|
||||||
|
|
||||||
|
## Shared glossary library
|
||||||
|
|
||||||
|
From `examples/campaigns`, `first.yml` and `second.yml` each consume the common
|
||||||
|
`shared.yml` through the companion `glossary.yml`. Inspect each configuration:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config first.yml
|
||||||
|
audita config validate --config second.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
With a local service configured, process either campaign:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process --config first.yml --output first-output.json transcript.json
|
||||||
|
audita process --config second.yml --output second-output.json transcript.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit `prompts.yml` for a local service before processing either campaign. The
|
||||||
|
first uses separate party and spell corrections; the second selects only party.
|
||||||
8
examples/campaigns/first.yml
Normal file
8
examples/campaigns/first.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm: {proposal_profile: local}
|
||||||
|
glossaries: [party, spells]
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- {id: names, module: glossary, glossaries: [party]}
|
||||||
|
- {id: magic, module: glossary, glossaries: [spells]}
|
||||||
|
- {module: grammar}
|
||||||
4
examples/campaigns/glossary.yml
Normal file
4
examples/campaigns/glossary.yml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: shared.yml
|
||||||
|
required: true
|
||||||
5
examples/campaigns/prompts.yml
Normal file
5
examples/campaigns/prompts.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
7
examples/campaigns/second.yml
Normal file
7
examples/campaigns/second.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
version: 3
|
||||||
|
llm: {proposal_profile: local}
|
||||||
|
glossaries: [party]
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- {id: names, module: glossary}
|
||||||
|
- {module: grammar}
|
||||||
12
examples/campaigns/shared.yml
Normal file
12
examples/campaigns/shared.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: lyra
|
||||||
|
name: Lyra
|
||||||
|
category: character
|
||||||
|
description: A fictional adventurer.
|
||||||
|
spells:
|
||||||
|
entries:
|
||||||
|
- id: shield
|
||||||
|
name: Shield
|
||||||
|
category: spell
|
||||||
|
aliases: [Ward]
|
||||||
9
examples/campaigns/transcript.json
Normal file
9
examples/campaigns/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "Lyra raises a ward before the journey."
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
version: 2
|
|
||||||
profiles_dir: profiles/local
|
|
||||||
backends:
|
|
||||||
local-base:
|
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
|
||||||
concurrency: 4
|
|
||||||
queue_capacity: 0
|
|
||||||
local:
|
|
||||||
extends: local-base
|
|
||||||
concurrency: 2
|
|
||||||
llm:
|
|
||||||
proposal_profile: local-proposal
|
|
||||||
validation_profile: local-validation
|
|
||||||
pipeline:
|
|
||||||
modules: [grammar]
|
|
||||||
8
examples/local/config.yml
Normal file
8
examples/local/config.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party]
|
||||||
7
examples/local/glossary.yml
Normal file
7
examples/local/glossary.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: jesters
|
||||||
|
name: Jesters
|
||||||
|
category: faction
|
||||||
|
description: A traveling group of performers.
|
||||||
|
aliases: [Jester]
|
||||||
5
examples/local/prompts.yml
Normal file
5
examples/local/prompts.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
9
examples/local/transcript.json
Normal file
9
examples/local/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
1
examples/local/validation-candidate.json
Normal file
1
examples/local/validation-candidate.json
Normal file
@@ -0,0 +1 @@
|
|||||||
|
[{"correction_index":1,"original_text":"not open","corrected_text":"open","original_segment_text":"The door is not open.","corrected_segment_text":"The door is open."}]
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
version: 2
|
|
||||||
profiles_dir: profiles/local
|
|
||||||
backends:
|
|
||||||
local:
|
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
|
||||||
concurrency: 1
|
|
||||||
llm:
|
|
||||||
proposal_profile: local-proposal
|
|
||||||
validation_profile: local-validation
|
|
||||||
pipeline:
|
|
||||||
modules: [grammar]
|
|
||||||
output:
|
|
||||||
schema: bare-segments
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
llm:
|
|
||||||
proposal_profile: service-proposal-testing
|
|
||||||
validation_profile: service-validation-testing
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
version: 2
|
|
||||||
profiles_dir: profiles/service
|
|
||||||
backends:
|
|
||||||
service:
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
credential: AUDITA_SERVICE_KEY
|
|
||||||
concurrency: 2
|
|
||||||
queue_capacity: 1024
|
|
||||||
llm:
|
|
||||||
proposal_profile: service-proposal
|
|
||||||
validation_profile: service-validation
|
|
||||||
transient_retries: 2
|
|
||||||
repair_attempts: 2
|
|
||||||
operation_timeout: 15m
|
|
||||||
pipeline:
|
|
||||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
|
||||||
output:
|
|
||||||
schema: audita-v1
|
|
||||||
chunking:
|
|
||||||
# Requires at least this many normalized segments; omit for automatic chunking.
|
|
||||||
target_sections: 8
|
|
||||||
max_section_tokens: 8192
|
|
||||||
min_section_tokens: 2048
|
|
||||||
normalization:
|
|
||||||
max_segment_gap: 4.0
|
|
||||||
ellipsis_gap: 3.5
|
|
||||||
max_segment_duration: 60.0
|
|
||||||
max_segment_tokens: 2048
|
|
||||||
thresholds:
|
|
||||||
glossary: 0.8
|
|
||||||
homophones: 0.8
|
|
||||||
spoken_word: 0.8
|
|
||||||
grammar: 0.8
|
|
||||||
context:
|
|
||||||
description: "General context for domain vocabulary and speaker names."
|
|
||||||
reporting:
|
|
||||||
enabled: true
|
|
||||||
stdout_dir: .
|
|
||||||
diagnostics:
|
|
||||||
debug: false
|
|
||||||
work_dir: /tmp/audita
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
local-proposal:
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
local-validation:
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
service-proposal-testing:
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
service-proposal:
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
service-validation-testing:
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
profiles:
|
|
||||||
service-validation:
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
8
examples/prompt-edit/config.yml
Normal file
8
examples/prompt-edit/config.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party]
|
||||||
7
examples/prompt-edit/glossary.yml
Normal file
7
examples/prompt-edit/glossary.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: jesters
|
||||||
|
name: Jesters
|
||||||
|
category: faction
|
||||||
|
description: A traveling group of performers.
|
||||||
|
aliases: [Jester]
|
||||||
23
examples/prompt-edit/prompts.yml
Normal file
23
examples/prompt-edit/prompts.yml
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
|
collections:
|
||||||
|
audita:
|
||||||
|
prompts:
|
||||||
|
- id: modules.grammar.proposal
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Correct only clearly supported grammatical errors.
|
||||||
|
- role: user
|
||||||
|
content: |
|
||||||
|
{{ .ProposalContract }}
|
||||||
|
{{ .HardeningPolicy }}
|
||||||
|
{{ .TranscriptDescriptionBlock }}
|
||||||
|
Glossary: {{ .GlossaryJSON }}
|
||||||
|
Transcript: {{ .SectionJSON }}
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: correction_set.v1.json
|
||||||
9
examples/prompt-edit/transcript.json
Normal file
9
examples/prompt-edit/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
7
examples/service/config.yml
Normal file
7
examples/service/config.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: service-proposal
|
||||||
|
validation_profile: service-review
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: grammar
|
||||||
3
examples/service/overlays/testing.yml
Normal file
3
examples/service/overlays/testing.yml
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
llm:
|
||||||
|
proposal_profile: service-test
|
||||||
|
validation_profile: service-test
|
||||||
16
examples/service/prompts.yml
Normal file
16
examples/service/prompts.yml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
backends:
|
||||||
|
- id: service
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
api_key_env: AUDITA_EXAMPLE_KEY
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 8
|
||||||
|
profiles:
|
||||||
|
- id: service-proposal
|
||||||
|
backend: service
|
||||||
|
model: example-proposal-model
|
||||||
|
- id: service-review
|
||||||
|
backend: service
|
||||||
|
model: example-review-model
|
||||||
|
- id: service-test
|
||||||
|
extends: {id: service-proposal}
|
||||||
|
model: example-testing-model
|
||||||
9
examples/service/transcript.json
Normal file
9
examples/service/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
glossary:
|
|
||||||
- name: Audita
|
|
||||||
aliases:
|
|
||||||
- audita
|
|
||||||
category: product
|
|
||||||
summary: The Audita transcript correction CLI.
|
|
||||||
20
go.mod
20
go.mod
@@ -1,25 +1,29 @@
|
|||||||
module gitea.maximumdirect.net/eric/audita/v3
|
module gitea.maximumdirect.net/eric/audita/v3
|
||||||
|
|
||||||
go 1.26.0
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
go.maximumdirect.net/ckconfig v0.7.0
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1
|
go.maximumdirect.net/ckprompt v0.7.0
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.14.0
|
go.maximumdirect.net/commandkit v0.6.0
|
||||||
golang.org/x/mod v0.40.0
|
go.maximumdirect.net/configkit v1.2.0
|
||||||
|
go.maximumdirect.net/promptkit v0.21.0
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
|
||||||
github.com/kr/pretty v0.3.1 // indirect
|
github.com/kr/pretty v0.3.1 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||||
|
github.com/spf13/cobra v1.10.1 // indirect
|
||||||
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
|
tool go.maximumdirect.net/releasekit/cmd/releasekit
|
||||||
|
|||||||
30
go.sum
30
go.sum
@@ -1,16 +1,9 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.14.0 h1:HUzlcrFk18dNGPBgnHqjAw87O0c9Wl/IC1xc/VmzL10=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.14.0/go.mod h1:2YGvNca3bNXZP+e+pJeGzsNUcxwkHvzvuoc+LNNiT3E=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
@@ -22,10 +15,27 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
|
|||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
|
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||||
|
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||||
|
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||||
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
|
go.maximumdirect.net/ckconfig v0.7.0 h1:jRmKqgReLYIRHJl7Pr4+TlucifFirGANcsoC+5WrW7A=
|
||||||
|
go.maximumdirect.net/ckconfig v0.7.0/go.mod h1:ZBKCi0Z72+C6kQV+1HJHXFGcZTc11FYyaH1pQDSZFl8=
|
||||||
|
go.maximumdirect.net/ckprompt v0.7.0 h1:iACE8fZIaq9+/DF8zt1C1JwVPgZh12F2xJmlin2r45U=
|
||||||
|
go.maximumdirect.net/ckprompt v0.7.0/go.mod h1:IuCIwxqKVjpZ+Rcd2JO2xmJ1QVtmPki5dV5PnzbD8CY=
|
||||||
|
go.maximumdirect.net/commandkit v0.6.0 h1:4PHhMIlZuuvreIxZhUazeweg6zoYixEvhM9LJnkbGNA=
|
||||||
|
go.maximumdirect.net/commandkit v0.6.0/go.mod h1:ZS98DewKNVBygKVDoh5MFD09CnPlP37dS9pVsID5Eoc=
|
||||||
|
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8=
|
||||||
|
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||||
|
go.maximumdirect.net/promptkit v0.21.0 h1:Ws4y28lnW+lIXAq4tGhXVFnQV/6AqkDChhg8knIZFrU=
|
||||||
|
go.maximumdirect.net/promptkit v0.21.0/go.mod h1:pi9acdCi80HWyzY+XYwRvMEoCiTcDWKyeBh3mzHdIFo=
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
|
|||||||
110
internal/cli/app_wiring.go
Normal file
110
internal/cli/app_wiring.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
"go.maximumdirect.net/ckprompt"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
"go.maximumdirect.net/releasekit/buildinfo"
|
||||||
|
)
|
||||||
|
|
||||||
|
type invocationState struct {
|
||||||
|
engines []engineEntry
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
type engineEntry struct {
|
||||||
|
selection ckconfig.Selection
|
||||||
|
engine *pk.Engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadApplication(ctx context.Context, selection ckconfig.Selection, _ struct{}, _ *invocationState) (*ck.Result[config.Config], error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var source ck.Source
|
||||||
|
if selection.Config.Set {
|
||||||
|
if strings.TrimSpace(selection.Config.Value) == "" {
|
||||||
|
return nil, fmt.Errorf("--config requires a nonempty path")
|
||||||
|
}
|
||||||
|
source = ck.File(selection.Config.Value)
|
||||||
|
} else {
|
||||||
|
source = ck.SearchConfig("audita", ck.Searches(ck.WorkingDirectory(), ck.UserConfigDirectory(), ck.UnixSystemDirectories()))
|
||||||
|
}
|
||||||
|
return config.Load(ck.Input{Source: source}, selection.Overlay.Value, selection.Overrides...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
loaded, err := loadApplication(ctx, selection, input, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var client *http.Client
|
||||||
|
if state != nil {
|
||||||
|
client = state.client
|
||||||
|
}
|
||||||
|
return llm.NewEngine(ctx, loaded, client)
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
if state != nil {
|
||||||
|
for _, entry := range state.engines {
|
||||||
|
if reflect.DeepEqual(entry.selection, selection) {
|
||||||
|
return entry.engine, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
engine, err := loadPromptEngine(ctx, selection, input, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state != nil {
|
||||||
|
state.engines = append(state.engines, engineEntry{selection: selection, engine: engine})
|
||||||
|
}
|
||||||
|
return engine, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildApplication(extra ...commandkit.Declaration) (*commandkit.App, error) {
|
||||||
|
return buildApplicationWithClient(nil, extra...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildApplicationWithClient(client *http.Client, extra ...commandkit.Declaration) (*commandkit.App, error) {
|
||||||
|
appScope := ckconfig.FromConfigKit("app", "Audita application settings.", loadApplication)
|
||||||
|
appScope.Features.Overrides = true
|
||||||
|
promptScope := ckprompt.ConfigScope(loadPromptEngine)
|
||||||
|
configCommand := ckconfig.Command(ckconfig.Suite[struct{}, *invocationState]{NewState: func() *invocationState { return &invocationState{client: client} }, Scopes: []ckconfig.Scope[struct{}, *invocationState]{appScope, promptScope}})
|
||||||
|
definitions := ckprompt.WithNamespaces(ckprompt.FromEngine(capturePromptEngine), pk.Namespaces{})
|
||||||
|
definitions.NewState = func() *invocationState { return &invocationState{client: client} }
|
||||||
|
definitions = ckprompt.WithRequestValidation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.ValidationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
|
||||||
|
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
|
||||||
|
})
|
||||||
|
definitions = ckprompt.WithRequestPreparation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.PreparationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
|
||||||
|
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
|
||||||
|
})
|
||||||
|
definitions = ckprompt.WithSmokeTesting(definitions, capturePromptEngine, nil)
|
||||||
|
declarations := []commandkit.Declaration{configCommand, commandkit.Completion("audita"), processCommand()}
|
||||||
|
declarations = append(declarations, ckprompt.Declarations(definitions)...)
|
||||||
|
declarations = append(declarations, extra...)
|
||||||
|
return commandkit.New(commandkit.Identity{Name: "audita", Version: buildinfo.Version()}, declarations...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// applicationPromptRequest supplies instructions only for Audita's maintained prompts.
|
||||||
|
func applicationPromptRequest(request pk.RunRequest, namespaces *pk.Namespaces) pk.RunRequest {
|
||||||
|
id := request.PromptID
|
||||||
|
if !strings.Contains(id, "/") && namespaces != nil && namespaces.Prompt != "" {
|
||||||
|
id = namespaces.Prompt + "/" + id
|
||||||
|
}
|
||||||
|
if _, ok := prompts.LookupMetadata(id); ok {
|
||||||
|
request.Vars = prompts.WithSharedVariables(request.Vars)
|
||||||
|
}
|
||||||
|
return request
|
||||||
|
}
|
||||||
232
internal/cli/app_wiring_test.go
Normal file
232
internal/cli/app_wiring_test.go
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
"go.maximumdirect.net/ckprompt"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNativeConfigurationAndResourceCommands(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(file, []byte("version: 3\ncontext:\n description: example\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
args []string
|
||||||
|
contains string
|
||||||
|
}{
|
||||||
|
{[]string{"--help"}, "audita"},
|
||||||
|
{[]string{"version"}, "audita"},
|
||||||
|
{[]string{"process", "--help"}, "transcript"},
|
||||||
|
{[]string{"config", "show", "--scope", "app", "--config", file}, "example"},
|
||||||
|
{[]string{"promptkit", "list", "prompts", "--config", file}, "audita/modules.grammar.proposal"},
|
||||||
|
} {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status != 0 || !strings.Contains(out.String(), tc.contains) {
|
||||||
|
t.Errorf("%v: status=%d out=%q err=%q", tc.args, result.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeConfigurationSelectionsAndComparison(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
|
||||||
|
for name, data := range map[string]string{first: "context:\n description: before\n", second: "context:\n description: after\n"} {
|
||||||
|
if err := os.WriteFile(name, []byte(data), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--scope=app", "--config", first, "--against-config", second, "--set=/context/description=\"left\"", "--against-set=/context/description=\"right\""}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status != 3 || !strings.Contains(out.String(), "left") || !strings.Contains(out.String(), "right") {
|
||||||
|
t.Fatalf("status=%d output=%q err=%q", outcome.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
outcome = app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config", filepath.Join(dir, "missing.yml")}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status == 0 {
|
||||||
|
t.Fatalf("missing file accepted: %s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExplicitEmptyConfigurationPathFails(t *testing.T) {
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config="}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status == 0 {
|
||||||
|
t.Fatal("explicit empty path accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptConfigurationComparisonAcquiresEqualSelectionsIndependently(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resource := filepath.Join(dir, "prompts.yml")
|
||||||
|
writeProfile := func(model string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(resource, []byte("profiles: [{id: local, model: "+model+"}]\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeProfile("before")
|
||||||
|
scope := ckprompt.ConfigScope(func(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
engine, err := loadPromptEngine(ctx, selection, input, state)
|
||||||
|
writeProfile("after")
|
||||||
|
return engine, err
|
||||||
|
})
|
||||||
|
side := ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{
|
||||||
|
Config: commandkit.Supplied[string]{Value: main, Set: true},
|
||||||
|
}}
|
||||||
|
result, err := scope.Compare(context.Background(), ckconfig.ComparisonRequest[struct{}]{Left: side, Right: side}, &invocationState{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, difference := range result.Value {
|
||||||
|
if strings.Contains(difference.Left.YAML, "before") && strings.Contains(difference.Right.YAML, "after") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("comparison did not capture the model before and after the resource changed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resources := `backends:
|
||||||
|
- id: local
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: fixture
|
||||||
|
authentication:
|
||||||
|
kind: none
|
||||||
|
profiles:
|
||||||
|
- id: fixture
|
||||||
|
backend: local
|
||||||
|
model: sample
|
||||||
|
collections:
|
||||||
|
audita:
|
||||||
|
prompts:
|
||||||
|
- id: modules.grammar.proposal
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
body := `{}`
|
||||||
|
if req.Method == http.MethodPost {
|
||||||
|
requestBody, err := io.ReadAll(req.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
content := "PROMPTKIT_OK"
|
||||||
|
if strings.Contains(string(requestBody), "response_format") {
|
||||||
|
content = `{"status":"ok"}`
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}`
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
||||||
|
})}
|
||||||
|
app, err := buildApplicationWithClient(client)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
execute := func(args ...string) (commandkit.Outcome, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
return result, out.String(), stderr.String()
|
||||||
|
}
|
||||||
|
common := []string{"--config", file, "--llm-profile", "fixture"}
|
||||||
|
args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...)
|
||||||
|
args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction")
|
||||||
|
outcome, out, stderr := execute(args...)
|
||||||
|
if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") {
|
||||||
|
t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...)
|
||||||
|
args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
||||||
|
outcome, out, stderr = execute(args...)
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...)
|
||||||
|
validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
||||||
|
outcome, out, stderr = execute(validatorPreview...)
|
||||||
|
if outcome.Status != 0 || !strings.Contains(out, "teh cat") {
|
||||||
|
t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
if calls != 0 {
|
||||||
|
t.Fatalf("inspection made %d provider calls", calls)
|
||||||
|
}
|
||||||
|
outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local")
|
||||||
|
if outcome.Status != 0 || calls == 0 {
|
||||||
|
t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
||||||
|
}
|
||||||
|
previous := calls
|
||||||
|
outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json")
|
||||||
|
if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") {
|
||||||
|
t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||||
|
|
||||||
|
func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) {
|
||||||
|
request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}}
|
||||||
|
result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"})
|
||||||
|
if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" {
|
||||||
|
t.Fatalf("variables=%v", result.Vars)
|
||||||
|
}
|
||||||
|
if _, changed := request.Vars["ProposalContract"]; changed {
|
||||||
|
t.Fatal("input variables mutated")
|
||||||
|
}
|
||||||
|
generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}}
|
||||||
|
result = applicationPromptRequest(generic, nil)
|
||||||
|
if len(result.Vars) != 1 || result.Vars["x"] != "y" {
|
||||||
|
t.Fatalf("generic request changed: %v", result.Vars)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
var systemConfigPaths = []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"}
|
|
||||||
|
|
||||||
type configInput struct {
|
|
||||||
ConfigPath string
|
|
||||||
ConfigPathSet bool
|
|
||||||
Overlay string
|
|
||||||
View string
|
|
||||||
TranscriptPath string
|
|
||||||
GlossaryPath string
|
|
||||||
OutputPath string
|
|
||||||
ReportDirSet bool
|
|
||||||
Overrides []ck.Override
|
|
||||||
}
|
|
||||||
|
|
||||||
type selectedConfig struct {
|
|
||||||
Path string
|
|
||||||
Source string
|
|
||||||
Result *ck.Result[config.Config]
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseConfigInput owns the common setting flags and process-only path flags.
|
|
||||||
// A transcript before flags and one after flags are both accepted.
|
|
||||||
func parseConfigInput(command string, args []string, output io.Writer) (configInput, bool, error) {
|
|
||||||
var input configInput
|
|
||||||
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
|
||||||
fs.SetOutput(output)
|
|
||||||
configPath := fs.String("config", "", "Main configuration YAML file")
|
|
||||||
overlay := fs.String("overlay", "", "Selected configuration overlay")
|
|
||||||
if command == "print-effective" {
|
|
||||||
fs.String("view", "settings", "Effective view: settings, profiles, or explain")
|
|
||||||
}
|
|
||||||
settingPaths := map[string]ck.Path{
|
|
||||||
"proposal-profile": ck.Keys("llm", "proposal_profile"),
|
|
||||||
"validation-profile": ck.Keys("llm", "validation_profile"),
|
|
||||||
"output-schema": ck.Keys("output", "schema"),
|
|
||||||
"validation-max-prompt-tokens": ck.Keys("validation", "max_prompt_tokens"),
|
|
||||||
"target-sections": ck.Keys("chunking", "target_sections"),
|
|
||||||
"max-section-tokens": ck.Keys("chunking", "max_section_tokens"),
|
|
||||||
"min-section-tokens": ck.Keys("chunking", "min_section_tokens"),
|
|
||||||
"normalize-max-segment-gap": ck.Keys("normalization", "max_segment_gap"),
|
|
||||||
"normalize-ellipsis-gap": ck.Keys("normalization", "ellipsis_gap"),
|
|
||||||
"normalize-max-segment-duration": ck.Keys("normalization", "max_segment_duration"),
|
|
||||||
"normalize-max-segment-tokens": ck.Keys("normalization", "max_segment_tokens"),
|
|
||||||
"glossary-confidence-threshold": ck.Keys("thresholds", "glossary"),
|
|
||||||
"homophones-confidence-threshold": ck.Keys("thresholds", "homophones"),
|
|
||||||
"spoken-word-confidence-threshold": ck.Keys("thresholds", "spoken_word"),
|
|
||||||
"grammar-confidence-threshold": ck.Keys("thresholds", "grammar"),
|
|
||||||
"transcript-description": ck.Keys("context", "description"),
|
|
||||||
"reporting": ck.Keys("reporting", "enabled"),
|
|
||||||
"report-dir": ck.Keys("reporting", "stdout_dir"),
|
|
||||||
"debug": ck.Keys("diagnostics", "debug"),
|
|
||||||
"work-dir": ck.Keys("diagnostics", "work_dir"),
|
|
||||||
"modules": ck.Keys("pipeline", "modules"),
|
|
||||||
}
|
|
||||||
for name := range settingPaths {
|
|
||||||
switch name {
|
|
||||||
case "reporting", "debug":
|
|
||||||
fs.Bool(name, false, "Setting override")
|
|
||||||
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
|
|
||||||
fs.Int(name, 0, "Setting override")
|
|
||||||
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
|
|
||||||
fs.Float64(name, 0, "Setting override")
|
|
||||||
default:
|
|
||||||
fs.String(name, "", "Setting override")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if command == "process" {
|
|
||||||
fs.String("glossary", "", "Glossary YAML file")
|
|
||||||
fs.String("output", "", "Corrected transcript output file")
|
|
||||||
}
|
|
||||||
parseArgs := args
|
|
||||||
front := ""
|
|
||||||
if command == "process" && len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
||||||
front, parseArgs = args[0], args[1:]
|
|
||||||
}
|
|
||||||
if err := fs.Parse(parseArgs); err != nil {
|
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
|
||||||
return configInput{}, true, nil
|
|
||||||
}
|
|
||||||
return configInput{}, false, err
|
|
||||||
}
|
|
||||||
positions := fs.Args()
|
|
||||||
if front != "" {
|
|
||||||
positions = append([]string{front}, positions...)
|
|
||||||
}
|
|
||||||
if command == "process" {
|
|
||||||
if len(positions) != 1 {
|
|
||||||
return configInput{}, false, fmt.Errorf("expected exactly one transcript JSON path")
|
|
||||||
}
|
|
||||||
input.TranscriptPath = positions[0]
|
|
||||||
input.GlossaryPath = fs.Lookup("glossary").Value.String()
|
|
||||||
input.OutputPath = fs.Lookup("output").Value.String()
|
|
||||||
if strings.TrimSpace(input.GlossaryPath) == "" {
|
|
||||||
return configInput{}, false, fmt.Errorf("--glossary is required")
|
|
||||||
}
|
|
||||||
} else if len(positions) != 0 {
|
|
||||||
return configInput{}, false, fmt.Errorf("unexpected positional arguments")
|
|
||||||
}
|
|
||||||
input.ConfigPath, input.Overlay = *configPath, *overlay
|
|
||||||
if command == "print-effective" {
|
|
||||||
input.View = fs.Lookup("view").Value.String()
|
|
||||||
switch input.View {
|
|
||||||
case "settings", "profiles", "explain":
|
|
||||||
default:
|
|
||||||
return configInput{}, false, fmt.Errorf("unknown effective view %q", input.View)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs.Visit(func(f *flag.Flag) {
|
|
||||||
if f.Name == "config" {
|
|
||||||
input.ConfigPathSet = true
|
|
||||||
}
|
|
||||||
if f.Name == "report-dir" {
|
|
||||||
input.ReportDirSet = true
|
|
||||||
}
|
|
||||||
path, ok := settingPaths[f.Name]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var value any = f.Value.String()
|
|
||||||
switch f.Name {
|
|
||||||
case "reporting", "debug":
|
|
||||||
value, _ = strconv.ParseBool(f.Value.String())
|
|
||||||
case "modules":
|
|
||||||
modules, err := config.ParseModulesCSV(f.Value.String())
|
|
||||||
if err != nil {
|
|
||||||
value = err
|
|
||||||
} else {
|
|
||||||
values := make([]any, len(modules))
|
|
||||||
for i, module := range modules {
|
|
||||||
values[i] = module
|
|
||||||
}
|
|
||||||
value = values
|
|
||||||
}
|
|
||||||
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
|
|
||||||
value, _ = strconv.Atoi(f.Value.String())
|
|
||||||
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
|
|
||||||
value, _ = strconv.ParseFloat(f.Value.String(), 64)
|
|
||||||
}
|
|
||||||
input.Overrides = append(input.Overrides, ck.Override{Path: path, Value: value, Flag: "--" + f.Name})
|
|
||||||
})
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
if err, ok := override.Value.(error); ok {
|
|
||||||
return configInput{}, false, fmt.Errorf("%s: %w", override.Flag, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return input, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectConfigPath(explicit string, set bool, stat func(string) (os.FileInfo, error), candidates []string) (string, string, error) {
|
|
||||||
if set {
|
|
||||||
if strings.TrimSpace(explicit) == "" {
|
|
||||||
return "", "", fmt.Errorf("--config requires a nonempty path")
|
|
||||||
}
|
|
||||||
if _, err := stat(explicit); err != nil {
|
|
||||||
return "", "", fmt.Errorf("config file %q: %w", explicit, err)
|
|
||||||
}
|
|
||||||
return explicit, "flag", nil
|
|
||||||
}
|
|
||||||
for _, path := range candidates {
|
|
||||||
if _, err := stat(path); err == nil {
|
|
||||||
return path, "default", nil
|
|
||||||
} else if !os.IsNotExist(err) {
|
|
||||||
return "", "", fmt.Errorf("config file %q: %w", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", "", fmt.Errorf("no configuration file found; select a config with backend and profile settings")
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadConfigInput(input configInput, command string) (selectedConfig, error) {
|
|
||||||
if command == "validate" && (!input.ConfigPathSet || strings.TrimSpace(input.ConfigPath) == "") {
|
|
||||||
return selectedConfig{}, fmt.Errorf("--config is required")
|
|
||||||
}
|
|
||||||
path, source, err := selectConfigPath(input.ConfigPath, input.ConfigPathSet, os.Stat, systemConfigPaths)
|
|
||||||
if err != nil {
|
|
||||||
return selectedConfig{}, err
|
|
||||||
}
|
|
||||||
loaded, err := config.Load(ck.File(path), input.Overlay, input.Overrides...)
|
|
||||||
if err != nil {
|
|
||||||
return selectedConfig{}, err
|
|
||||||
}
|
|
||||||
return selectedConfig{Path: path, Source: source, Result: loaded}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func configInputError(err error) string {
|
|
||||||
var loadErr *ck.LoadError
|
|
||||||
if !errors.As(err, &loadErr) {
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
var messages []string
|
|
||||||
for _, diagnostic := range loadErr.Diagnostics() {
|
|
||||||
messages = append(messages, diagnostic.Message)
|
|
||||||
}
|
|
||||||
if len(messages) == 0 {
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
return strings.Join(messages, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectedReportDir(cfg config.Config, outputPath string, reportDirSet bool) (string, error) {
|
|
||||||
if !cfg.Reporting.Enabled {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
var reportDir string
|
|
||||||
if outputPath != "" {
|
|
||||||
if reportDirSet {
|
|
||||||
reportDir = cfg.Reporting.StdoutDir
|
|
||||||
} else {
|
|
||||||
reportDir = filepath.Dir(outputPath)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
reportDir = cfg.Reporting.StdoutDir
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(reportDir) == "" {
|
|
||||||
return "", fmt.Errorf("report directory is required when reporting is enabled for the selected destination")
|
|
||||||
}
|
|
||||||
return reportDir, nil
|
|
||||||
}
|
|
||||||
@@ -1,280 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSharedConfigInput(t *testing.T) {
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
t.Run(command, func(t *testing.T) {
|
|
||||||
args := []string{"--config", "main.yml", "--overlay", "testing", "--proposal-profile", "economy", "--modules", "", "--target-sections", "0"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, help, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil || help {
|
|
||||||
t.Fatalf("parse: %+v %v %v", input, help, err)
|
|
||||||
}
|
|
||||||
if !input.ConfigPathSet || input.ConfigPath != "main.yml" || input.Overlay != "testing" || len(input.Overrides) != 3 {
|
|
||||||
t.Fatalf("selection: %+v", input)
|
|
||||||
}
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
switch override.Path.String() {
|
|
||||||
case ck.Keys("pipeline", "modules").String():
|
|
||||||
if values, ok := override.Value.([]any); !ok || len(values) != 0 {
|
|
||||||
t.Fatalf("empty modules: %#v", override.Value)
|
|
||||||
}
|
|
||||||
case ck.Keys("chunking", "target_sections").String():
|
|
||||||
if override.Value != 0 {
|
|
||||||
t.Fatalf("zero target: %#v", override.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigInputRejectsRemovedAndProcessOnlyFlags(t *testing.T) {
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
for _, flag := range []string{"--model", "--llm-api-key", "--max-retries", "--llm-concurrency", "--report-json", "--work-dir-retention"} {
|
|
||||||
args := []string{flag, "value"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput(command, args, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted %s", command, flag)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, command := range []string{"validate", "print-effective"} {
|
|
||||||
if _, _, err := parseConfigInput(command, []string{"--glossary", "glossary.yml"}, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted process path", command)
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput(command, []string{"transcript.json"}, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted positional", command)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
_, help, err := parseConfigInput(command, []string{"--help"}, io.Discard)
|
|
||||||
if err != nil || !help {
|
|
||||||
t.Fatalf("%s help: %v %v", command, help, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigSelection(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
|
|
||||||
if err := os.WriteFile(second, []byte("version: 2\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path, source, err := selectConfigPath("", false, os.Stat, []string{first, second})
|
|
||||||
if err != nil || path != second || source != "default" {
|
|
||||||
t.Fatalf("discovery: %q %q %v", path, source, err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
explicit string
|
|
||||||
set bool
|
|
||||||
candidates []string
|
|
||||||
}{
|
|
||||||
{"", true, []string{second}}, {first, true, []string{second}}, {"", false, []string{first}},
|
|
||||||
} {
|
|
||||||
if _, _, err := selectConfigPath(tc.explicit, tc.set, os.Stat, tc.candidates); err == nil {
|
|
||||||
t.Fatalf("accepted unavailable selection: %+v", tc)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
statDenied := func(string) (os.FileInfo, error) { return nil, os.ErrPermission }
|
|
||||||
if _, _, err := selectConfigPath("", false, statDenied, []string{first}); !errors.Is(err, os.ErrPermission) {
|
|
||||||
t.Fatalf("inaccessible candidate: %v", err)
|
|
||||||
}
|
|
||||||
path, source, err = selectConfigPath(second, true, os.Stat, []string{first})
|
|
||||||
if err != nil || path != second || source != "flag" {
|
|
||||||
t.Fatalf("explicit path: %q %q %v", path, source, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) {
|
|
||||||
t.Setenv("AUDITA_CONFIG", filepath.Join(t.TempDir(), "legacy.yml"))
|
|
||||||
_, _, err := selectConfigPath("", false, os.Stat, []string{filepath.Join(t.TempDir(), "missing.yml")})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "no configuration file") {
|
|
||||||
t.Fatalf("legacy environment changed discovery: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEffectiveViewParsing(t *testing.T) {
|
|
||||||
for _, view := range []string{"settings", "profiles", "explain"} {
|
|
||||||
input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard)
|
|
||||||
if err != nil || input.View != view {
|
|
||||||
t.Fatalf("%q: %+v %v", view, input, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil {
|
|
||||||
t.Fatal("unknown view accepted")
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil {
|
|
||||||
t.Fatal("validate accepted inspection-only flag")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
main := filepath.Join(dir, "config.yml")
|
|
||||||
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\nchunking:\n max_section_tokens: 900\n"
|
|
||||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
args := []string{"--config", main, "--overlay", "testing", "--min-section-tokens", "500"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, command)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c := selected.Result.Config
|
|
||||||
if c.LLM.ProposalProfile != "testing" || c.LLM.ValidationProfile != "testing" || c.Chunking.MinSectionTokens != 500 || c.Chunking.MaxSectionTokens != 900 {
|
|
||||||
t.Fatalf("%s resolved %+v", command, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput("validate", []string{"--overlay", "testing"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := loadConfigInput(input, "validate"); err == nil {
|
|
||||||
t.Fatal("validate accepted discovery")
|
|
||||||
}
|
|
||||||
input, _, err = parseConfigInput("validate", []string{"--config", main}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := loadConfigInput(input, "validate"); err == nil {
|
|
||||||
t.Fatal("unmodified invalid lower-precedence section relation accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
main := filepath.Join(dir, "config.yml")
|
|
||||||
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\n"
|
|
||||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("reporting:\n enabled: true\n stdout_dir: reports\ndiagnostics:\n debug: true\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defaultInput, _, err := parseConfigInput("validate", []string{"--config", main}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defaults, err := loadConfigInput(defaultInput, "validate")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !defaults.Result.Config.Reporting.Enabled || defaults.Result.Config.Reporting.StdoutDir != "." || defaults.Result.Config.Diagnostics.Debug {
|
|
||||||
t.Fatalf("unexpected defaults: %+v", defaults.Result.Config)
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
args := []string{"--config", main, "--overlay", "testing", "--reporting=false", "--debug=false"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(input.Overrides) != 2 || input.Overrides[0].Value != false || input.Overrides[1].Value != false {
|
|
||||||
t.Fatalf("boolean overrides must remain typed: %+v", input.Overrides)
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, command)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if selected.Result.Config.Reporting.Enabled || selected.Result.Config.Diagnostics.Debug || selected.Result.Config.Reporting.StdoutDir != "reports" {
|
|
||||||
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
processConfig, err := loadConfigInput(processInput, "process")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" {
|
|
||||||
t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
|
|
||||||
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
|
|
||||||
if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" {
|
|
||||||
t.Fatalf("stdout destination: %q %v", got, err)
|
|
||||||
}
|
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
|
||||||
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
|
|
||||||
}
|
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "reports" {
|
|
||||||
t.Fatalf("explicit file report directory: %q %v", got, err)
|
|
||||||
}
|
|
||||||
cfg.Reporting.StdoutDir = ""
|
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
|
|
||||||
t.Fatalf("inactive blank configured report directory rejected for file output: %q %v", got, err)
|
|
||||||
}
|
|
||||||
if _, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err == nil {
|
|
||||||
t.Fatal("active explicit blank report directory accepted for file output")
|
|
||||||
}
|
|
||||||
if _, err := selectedReportDir(cfg, "", false); err == nil {
|
|
||||||
t.Fatal("enabled stdout reporting accepted blank destination")
|
|
||||||
}
|
|
||||||
cfg.Reporting.Enabled = false
|
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "" {
|
|
||||||
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
|
|
||||||
input, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--reporting", "--debug", "--report-dir", "reports"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
values := map[string]any{}
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
values[override.Path.String()] = override.Value
|
|
||||||
}
|
|
||||||
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
|
|
||||||
t.Fatalf("incorrect visited overrides: %+v", values)
|
|
||||||
}
|
|
||||||
if !input.ReportDirSet {
|
|
||||||
t.Fatal("explicit --report-dir presence was not recorded")
|
|
||||||
}
|
|
||||||
withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if withoutReportDir.ReportDirSet {
|
|
||||||
t.Fatal("absent --report-dir was recorded as explicit")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
156
internal/cli/examples_integration_test.go
Normal file
156
internal/cli/examples_integration_test.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
|
||||||
|
_, file, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("cannot locate examples")
|
||||||
|
}
|
||||||
|
root := filepath.Join(filepath.Dir(file), "..", "..", "examples")
|
||||||
|
t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key")
|
||||||
|
for _, tc := range []struct{ dir, main, overlay string }{
|
||||||
|
{"local", "config.yml", ""}, {"service", "config.yml", "testing"},
|
||||||
|
{"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""},
|
||||||
|
} {
|
||||||
|
t.Run(tc.dir+"/"+tc.main, func(t *testing.T) {
|
||||||
|
main := filepath.Join(root, tc.dir, tc.main)
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded.Config.Reporting.Enabled = false
|
||||||
|
var calls atomic.Int32
|
||||||
|
var mu sync.Mutex
|
||||||
|
var models, prompts []string
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var content strings.Builder
|
||||||
|
for _, message := range request.Messages {
|
||||||
|
content.WriteString(message.Content)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
models = append(models, request.Model)
|
||||||
|
prompts = append(prompts, content.String())
|
||||||
|
mu.Unlock()
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil
|
||||||
|
})
|
||||||
|
engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if tc.dir == "prompt-edit" {
|
||||||
|
prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
|
||||||
|
if err != nil || !strings.Contains(prompt.Messages[0].Content, "clearly supported grammatical errors") {
|
||||||
|
t.Fatalf("operator prompt: %+v %v", prompt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source}
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
|
||||||
|
t.Fatalf("provider calls=%d", calls.Load())
|
||||||
|
}
|
||||||
|
if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") {
|
||||||
|
t.Fatalf("overlay models=%v", models)
|
||||||
|
}
|
||||||
|
if tc.dir == "campaigns" && tc.main == "first.yml" {
|
||||||
|
if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) {
|
||||||
|
t.Fatalf("campaign glossary selections: %v", prompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.dir == "campaigns" && tc.main == "second.yml" {
|
||||||
|
if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) {
|
||||||
|
t.Fatalf("second campaign glossary selection: %v", prompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) {
|
||||||
|
_, file, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("cannot locate examples")
|
||||||
|
}
|
||||||
|
source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit")
|
||||||
|
dir := t.TempDir()
|
||||||
|
for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(source, name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inspect := func() string {
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := llm.NewRuntime(context.Background(), loaded, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return prompt.Messages[0].Content
|
||||||
|
}
|
||||||
|
before := inspect()
|
||||||
|
filePath := filepath.Join(dir, "prompts.yml")
|
||||||
|
raw, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after := inspect()
|
||||||
|
if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") {
|
||||||
|
t.Fatalf("before=%q after=%q", before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,442 +1,92 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
|
||||||
coreio "gitea.maximumdirect.net/eric/audita/v3/internal/core/io"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
|
"go.maximumdirect.net/ckconfig"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
ck "go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
// processResult carries the runner's transcript and ordinary report facts.
|
||||||
input, help, err := parseConfigInput("process", args, stderr)
|
type processResult struct {
|
||||||
if err != nil {
|
Transcript *schema.Transcript
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
|
Snapshot reporting.Snapshot
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if help {
|
|
||||||
writeProcessUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, "process")
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
cfg := selected.Result.Config
|
|
||||||
reportDir, err := selectedReportDir(cfg, input.OutputPath, input.ReportDirSet)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if cfg.Diagnostics.Debug && strings.TrimSpace(cfg.Diagnostics.WorkDir) == "" {
|
|
||||||
fmt.Fprintln(stderr, "audita process: debug work directory is required")
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
runtime, err := llm.NewRuntime(ctx, cfg, selected.Path, nil)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
specs, err := contracts.ResolveModuleRunSpecs(cfg.Pipeline.Modules)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
runID, err := generateRunID()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: run ID: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
started := time.Now().UTC()
|
|
||||||
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
|
|
||||||
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
|
|
||||||
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
|
|
||||||
protected := []string{input.GlossaryPath}
|
|
||||||
for _, source := range selected.Result.Sources() {
|
|
||||||
if source.Path != "" {
|
|
||||||
protected = append(protected, source.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
protected = append(protected, runtime.SourcePaths...)
|
|
||||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
|
||||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
state.session = session
|
|
||||||
initial := state.snapshot(reporting.StatusInProgress, nil, nil)
|
|
||||||
initialBundle, err := state.bundle(initial)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
initFailure := []processreport.RunFailure{{Phase: "initialization", Reason: outcomes.ReasonInternalFailure}}
|
|
||||||
failedInit, err := state.bundle(state.snapshot(reporting.StatusFailed, nil, initFailure))
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if err := session.Initialize(initialBundle, failedInit); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
state.startDebug()
|
|
||||||
return state.execute()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type processState struct {
|
func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcome {
|
||||||
cfg config.Config
|
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
|
||||||
runtime *llm.Runtime
|
|
||||||
specs []contracts.ModuleRunSpec
|
|
||||||
runID string
|
|
||||||
started time.Time
|
|
||||||
transcriptPath string
|
|
||||||
glossaryPath string
|
|
||||||
outputPath string
|
|
||||||
selected selectedConfig
|
|
||||||
overlay string
|
|
||||||
ctx context.Context
|
|
||||||
stdout io.Writer
|
|
||||||
stderr io.Writer
|
|
||||||
session *diagnostics.OutputSession
|
|
||||||
debugDir *diagnostics.RunDirectory
|
|
||||||
debugWarnings []string
|
|
||||||
debugMu sync.Mutex
|
|
||||||
runOutput *runner.RunOutput
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateRunID() (string, error) {
|
|
||||||
var random [12]byte
|
|
||||||
if _, err := rand.Read(random[:]); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return "run-" + hex.EncodeToString(random[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
|
|
||||||
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
|
|
||||||
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
|
||||||
OutputPath: s.outputPath, ModuleSpecs: s.specs, RunOutput: s.runOutput, RunFailures: failures})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) bundle(snapshot reporting.Snapshot) (diagnostics.ReportBundle, error) {
|
|
||||||
if !s.cfg.Reporting.Enabled {
|
|
||||||
return diagnostics.ReportBundle{}, nil
|
|
||||||
}
|
|
||||||
secrets := llm.ConfiguredSecrets(&s.cfg)
|
|
||||||
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument(), secrets...)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diagnostics.ReportBundle{}, err
|
if inv.Context.Err() != nil || llm.IsAssetError(err) {
|
||||||
|
return commandkit.Failure("Could not prepare processing.").WithCause(err)
|
||||||
}
|
}
|
||||||
recoveriesYAML, err := reporting.YAML(snapshot.RecoveriesDocument(), secrets...)
|
var loadErr *ck.LoadError
|
||||||
|
if errors.As(err, &loadErr) {
|
||||||
|
_, explanation := ckconfig.AdaptDiagnostics("Application configuration:\n", loadErr.Sources(), loadErr.Diagnostics())
|
||||||
|
return commandkit.Usage(loadErr.Error(), explanation.Details...).WithCause(err)
|
||||||
|
}
|
||||||
|
return commandkit.Usage(err.Error()).WithCause(err)
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diagnostics.ReportBundle{}, err
|
return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err)
|
||||||
}
|
}
|
||||||
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML,
|
defer prepared.Prepared.Output.Close()
|
||||||
Markdown: reporting.Markdown(snapshot, secrets...)}, nil
|
var warningsMu sync.Mutex
|
||||||
}
|
warn := func(err error) {
|
||||||
|
|
||||||
func (s *processState) debugWrite(label string, write func(*diagnostics.RunDirectory) error) {
|
|
||||||
if s.debugDir == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := write(s.debugDir); err != nil {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("%s: %w", label, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) startDebug() {
|
|
||||||
if !s.cfg.Diagnostics.Debug {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
dir, err := diagnostics.NewRunDirectory(s.cfg.Diagnostics.WorkDir, s.runID, llm.ConfiguredSecrets(&s.cfg)...)
|
|
||||||
if err != nil {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("directory: %w", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.debugDir = dir
|
|
||||||
s.debugWrite("invocation", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
|
||||||
Operation: "process", TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
|
||||||
OutputPath: s.outputPath, ConfigPath: s.selected.Path, ConfigSource: s.selected.Source,
|
|
||||||
Overlay: s.overlay, ConfigVersion: &s.cfg.Version, TranscriptDescription: s.cfg.Context.Description,
|
|
||||||
Modules: append([]string(nil), s.cfg.Pipeline.Modules...), RunID: s.runID, StartedAt: s.started,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
var effective bytes.Buffer
|
|
||||||
if err := s.selected.Result.WriteEffective(&effective); err == nil {
|
|
||||||
s.debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) })
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("effective settings: %w", err))
|
|
||||||
}
|
|
||||||
if origins, err := llm.ExplanationYAML(s.selected.Result); err == nil {
|
|
||||||
s.debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("configuration origins: %w", err))
|
|
||||||
}
|
|
||||||
if profiles, err := s.runtime.SelectedProfilesYAML(); err == nil {
|
|
||||||
s.debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("selected profiles: %w", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) debugPath() string {
|
|
||||||
if s.debugDir == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return s.debugDir.Path()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) execute() int {
|
|
||||||
transcriptBytes, err := coreio.ReadRequiredFile(s.transcriptPath, "transcript")
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
glossaryBytes, err := coreio.ReadRequiredFile(s.glossaryPath, "glossary")
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
source, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInputInvalid, err)
|
|
||||||
}
|
|
||||||
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInputInvalid, err)
|
|
||||||
}
|
|
||||||
s.debugWrite("source transcript", func(d *diagnostics.RunDirectory) error { return d.WriteSourceTranscript(source, transcriptBytes) })
|
|
||||||
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
|
||||||
MaxSegmentGap: s.cfg.Normalization.MaxSegmentGap, EllipsisGap: s.cfg.Normalization.EllipsisGap,
|
|
||||||
MaxSegmentDuration: s.cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: s.cfg.Normalization.MaxSegmentTokens,
|
|
||||||
})
|
|
||||||
normalized, normSummary := normalizer.Normalize(sourceToCanonicalTranscript(source))
|
|
||||||
s.debugWrite("normalized transcript", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizedTranscript(normalized) })
|
|
||||||
s.debugWrite("normalization summary", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizationSummary(normSummary) })
|
|
||||||
if err := s.ctx.Err(); err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonCanceled, err)
|
|
||||||
}
|
|
||||||
observations := llm.NewObservationCollector()
|
|
||||||
proposalClient, err := s.runtime.NewRoleClient("proposal", observations.Hooks())
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
validationClient, err := s.runtime.NewRoleClient("validation", observations.Hooks())
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
factory := modules.NewFactory(modules.Dependencies{Config: &s.cfg, Glossary: glossary, DiagnosticsDir: s.debugPath()})
|
|
||||||
result, runErr := runner.New(factory).Run(s.ctx, runner.RunInput{
|
|
||||||
Config: &s.cfg, Transcript: normalized, Glossary: glossary, ModuleSpecs: s.specs,
|
|
||||||
ProposalLLMClient: proposalClient, ProposalDiagnosticsDir: s.debugPath(),
|
|
||||||
ValidationLLMClient: validationClient, ValidationDiagnosticsDir: s.debugPath(),
|
|
||||||
CaptureCandidates: s.debugDir != nil,
|
|
||||||
ReportDiagnosticError: s.reportDiagnosticError,
|
|
||||||
})
|
|
||||||
result.Observations = observations.Snapshot()
|
|
||||||
result.ObservationsAvailable = true
|
|
||||||
result.LLMSummary = llm.Summarize(result.Observations)
|
|
||||||
s.runOutput = &result
|
|
||||||
s.writeExecutionEvidence()
|
|
||||||
if len(result.Sections) > 0 || runErr == nil {
|
|
||||||
s.writeChunkSummary(result.Sections)
|
|
||||||
}
|
|
||||||
if runErr != nil {
|
|
||||||
return s.fail("processing", fatalReason(s.ctx, runErr), runErr)
|
|
||||||
}
|
|
||||||
encoder, err := outputschema.Resolve(s.cfg.Output.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("encoding", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
outputBytes, err := encoder.Encoder(result.FinalTranscript)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("encoding", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
return s.finish(outputBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) writeChunkSummary(inventory []contracts.SectionMetadata) {
|
|
||||||
if s.debugDir == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sections := make([]chunking.Section, len(inventory))
|
|
||||||
for i, metadata := range inventory {
|
|
||||||
sections[i] = chunking.Section{Index: metadata.Index, StartSegmentID: metadata.StartSegmentID,
|
|
||||||
EndSegmentID: metadata.EndSegmentID, EstimatedTokens: metadata.EstimatedTokens,
|
|
||||||
Segments: make([]schema.Segment, metadata.SegmentCount)}
|
|
||||||
}
|
|
||||||
config := chunking.ChunkingConfig{TargetSections: s.cfg.Chunking.TargetSections,
|
|
||||||
MaxSectionTokens: s.cfg.Chunking.MaxSectionTokens, MinSectionTokens: s.cfg.Chunking.MinSectionTokens}
|
|
||||||
summary := chunking.ComputeDetailedSummary(sections, config)
|
|
||||||
s.debugWrite("chunking summary", func(d *diagnostics.RunDirectory) error { return d.WriteChunkingSummary(&summary) })
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) writeExecutionEvidence() {
|
|
||||||
if s.debugDir == nil || s.runOutput == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
output := s.runOutput
|
|
||||||
s.debugWrite("model execution", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations))
|
|
||||||
})
|
|
||||||
s.debugWrite("utilization", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, s.cfg.Backends))
|
|
||||||
})
|
|
||||||
s.debugWrite("correction ledger", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
|
||||||
RunDirectoryPath: d.Path(), RunOutput: output}))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func fatalReason(ctx context.Context, err error) outcomes.ReasonCode {
|
|
||||||
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
|
|
||||||
return outcomes.ReasonCanceled
|
|
||||||
}
|
|
||||||
var failure *structuredoutput.Failure
|
|
||||||
if errors.As(err, &failure) && failure.Reason != "" {
|
|
||||||
return failure.Reason
|
|
||||||
}
|
|
||||||
return outcomes.ReasonInternalFailure
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) fail(phase string, reason outcomes.ReasonCode, cause error) int {
|
|
||||||
s.debugWrite("error log", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteErrorLog(phase + ": " + llm.RedactError(cause, s.cfg))
|
|
||||||
})
|
|
||||||
failures := []processreport.RunFailure{{Phase: phase, Reason: reason}}
|
|
||||||
completed := time.Now().UTC()
|
|
||||||
base, err := s.bundle(s.snapshot(reporting.StatusFailed, &completed, failures))
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(err, s.cfg))
|
|
||||||
}
|
|
||||||
writeFailures := s.session.Finalize(nil, base, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
|
|
||||||
all := append(append([]processreport.RunFailure(nil), failures...), mapWriteFailures(extra)...)
|
|
||||||
bundle, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, all))
|
|
||||||
if renderErr != nil {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return bundle
|
|
||||||
})
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(cause, s.cfg))
|
|
||||||
s.printFailures(writeFailures)
|
|
||||||
s.printDebugWarnings()
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) finish(outputBytes []byte) int {
|
|
||||||
completed := time.Now().UTC()
|
|
||||||
snapshot := s.snapshot(reporting.StatusCompleted, &completed, nil)
|
|
||||||
complete, err := s.bundle(snapshot)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("report_write", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
if s.session.TranscriptPath() == "" {
|
|
||||||
if _, err := s.stdout.Write(outputBytes); err != nil {
|
|
||||||
return s.fail("transcript_write", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
outputBytes = nil
|
|
||||||
}
|
|
||||||
writeFailures := s.session.Finalize(outputBytes, complete, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
|
|
||||||
failed, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, mapWriteFailures(extra)))
|
|
||||||
if renderErr != nil {
|
|
||||||
return diagnostics.ReportBundle{}
|
|
||||||
}
|
|
||||||
return failed
|
|
||||||
})
|
|
||||||
if len(writeFailures) > 0 {
|
|
||||||
s.printFailures(writeFailures)
|
|
||||||
s.printDebugWarnings()
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
fmt.Fprintln(s.stderr, reporting.CompletionLine(snapshot))
|
|
||||||
for _, path := range s.session.ReportPaths() {
|
|
||||||
fmt.Fprintf(s.stderr, "Report: %s\n", s.safePath(path))
|
|
||||||
}
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "Debug: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
s.printDebugWarnings()
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapWriteFailures(failures []diagnostics.FileFailure) []processreport.RunFailure {
|
|
||||||
out := make([]processreport.RunFailure, len(failures))
|
|
||||||
for i, failure := range failures {
|
|
||||||
phase := "report_write"
|
|
||||||
if failure.Kind == "transcript write" {
|
|
||||||
phase = "transcript_write"
|
|
||||||
}
|
|
||||||
out[i] = processreport.RunFailure{Phase: phase, Reason: outcomes.ReasonInternalFailure, OutputPath: failure.Path}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) printFailures(failures []diagnostics.FileFailure) {
|
|
||||||
for _, failure := range failures {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(failure, s.cfg))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) printDebugWarnings() {
|
|
||||||
s.debugMu.Lock()
|
|
||||||
defer s.debugMu.Unlock()
|
|
||||||
for _, warning := range s.debugWarnings {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", warning)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// safePath sanitizes presentation only; filesystem destinations remain literal.
|
|
||||||
func (s *processState) safePath(path string) string {
|
|
||||||
return redaction.New(llm.ConfiguredSecrets(&s.cfg)).String(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) reportDiagnosticError(err error) {
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
message := "debug " + llm.RedactError(err, s.cfg)
|
warningsMu.Lock()
|
||||||
s.debugMu.Lock()
|
prepared.Prepared.Warnings = append(prepared.Prepared.Warnings, err)
|
||||||
defer s.debugMu.Unlock()
|
warningsMu.Unlock()
|
||||||
s.debugWarnings = append(s.debugWarnings, message)
|
}
|
||||||
|
debug := prepared.Prepared.Debug
|
||||||
|
if debug != nil {
|
||||||
|
warn(debug.WriteSourceTranscript(loaded.Source, loaded.RawTranscript))
|
||||||
|
}
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
||||||
|
MaxSegmentGap: cfg.Normalization.MaxSegmentGap, EllipsisGap: cfg.Normalization.EllipsisGap,
|
||||||
|
MaxSegmentDuration: cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: cfg.Normalization.MaxSegmentTokens,
|
||||||
|
})
|
||||||
|
normalized, summary := normalizer.Normalize(sourceToCanonicalTranscript(loaded.Source))
|
||||||
|
if debug != nil {
|
||||||
|
warn(debug.WriteNormalizedTranscript(normalized))
|
||||||
|
warn(debug.WriteNormalizationSummary(summary))
|
||||||
|
}
|
||||||
|
if err := inv.Context.Err(); err != nil {
|
||||||
|
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err)
|
||||||
|
}
|
||||||
|
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
|
||||||
|
if runErr != nil {
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr)
|
||||||
|
}
|
||||||
|
encoder, err := outputschema.Resolve(cfg.Output.Schema)
|
||||||
|
if err != nil {
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
|
||||||
|
}
|
||||||
|
if result.Transcript == nil {
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
|
||||||
|
}
|
||||||
|
payload, err := encoder.Encoder(result.Transcript)
|
||||||
|
if err != nil {
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
|
||||||
|
}
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugPath returns the retained artifact directory, if its preparation succeeded.
|
||||||
|
func (p *preparedProcess) DebugPath() string {
|
||||||
|
if p == nil || p.Prepared == nil || p.Prepared.Debug == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return p.Prepared.Debug.Path()
|
||||||
}
|
}
|
||||||
|
|||||||
120
internal/cli/process_command.go
Normal file
120
internal/cli/process_command.go
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processInput struct {
|
||||||
|
Config commandkit.Supplied[string]
|
||||||
|
Overlay commandkit.Supplied[string]
|
||||||
|
Transcript string
|
||||||
|
Output commandkit.Supplied[string]
|
||||||
|
Debug commandkit.Supplied[bool]
|
||||||
|
Overrides []ck.Override
|
||||||
|
}
|
||||||
|
|
||||||
|
func processCommand() commandkit.Command {
|
||||||
|
stringValue := func(s string) (string, error) { return s, nil }
|
||||||
|
flags := []commandkit.Flag[processInput]{
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Application configuration file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Config = v }),
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Overlay = v }),
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "output", Usage: "Transcript output file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Output = v }),
|
||||||
|
commandkit.BoolFlag(commandkit.FlagMeta{Name: "debug", Usage: "Capture diagnostic artifacts."}, false, func(in *processInput, v commandkit.Supplied[bool]) { in.Debug = v }),
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"proposal-profile", ck.Keys("llm", "proposal_profile")}, {"validation-profile", ck.Keys("llm", "validation_profile")},
|
||||||
|
{"output-schema", ck.Keys("output", "schema")}, {"transcript-description", ck.Keys("context", "description")},
|
||||||
|
{"report-dir", ck.Keys("reporting", "stdout_dir")}, {"work-dir", ck.Keys("diagnostics", "work_dir")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*string)(nil), stringValue,
|
||||||
|
func(in *processInput, v commandkit.Supplied[string]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"validation-max-prompt-tokens", ck.Keys("validation", "max_prompt_tokens")}, {"target-sections", ck.Keys("chunking", "target_sections")},
|
||||||
|
{"max-section-tokens", ck.Keys("chunking", "max_section_tokens")}, {"min-section-tokens", ck.Keys("chunking", "min_section_tokens")},
|
||||||
|
{"normalize-max-segment-tokens", ck.Keys("normalization", "max_segment_tokens")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*int)(nil), strconv.Atoi,
|
||||||
|
func(in *processInput, v commandkit.Supplied[int]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"normalize-max-segment-gap", ck.Keys("normalization", "max_segment_gap")}, {"normalize-ellipsis-gap", ck.Keys("normalization", "ellipsis_gap")},
|
||||||
|
{"normalize-max-segment-duration", ck.Keys("normalization", "max_segment_duration")}, {"glossary-confidence-threshold", ck.Keys("thresholds", "glossary")},
|
||||||
|
{"homophones-confidence-threshold", ck.Keys("thresholds", "homophones")}, {"spoken-word-confidence-threshold", ck.Keys("thresholds", "spoken_word")},
|
||||||
|
{"grammar-confidence-threshold", ck.Keys("thresholds", "grammar")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*float64)(nil), func(s string) (float64, error) { return strconv.ParseFloat(s, 64) },
|
||||||
|
func(in *processInput, v commandkit.Supplied[float64]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"glossaries", ck.Keys("glossaries")}, {"protected-terms", ck.Keys("protected_terms")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "JSON array of collection names, or null."}, (*any)(nil), parseCollectionNames,
|
||||||
|
func(in *processInput, value commandkit.Supplied[any]) {
|
||||||
|
if value.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: value.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
flags = append(flags, commandkit.BoolFlag(commandkit.FlagMeta{Name: "reporting", Usage: "Enable ordinary reports."}, false,
|
||||||
|
func(in *processInput, v commandkit.Supplied[bool]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: ck.Keys("reporting", "enabled"), Value: v.Value, Flag: "--reporting"})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
return commandkit.NewCommand(commandkit.CommandSpec[processInput]{Name: "process", Summary: "Polish a transcript.", Flags: flags, Run: runProcess,
|
||||||
|
Arguments: []commandkit.Argument[processInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "transcript", Usage: "Transcript JSON file."}, true, stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Transcript = v.Value })},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseCollectionNames(raw string) (any, error) {
|
||||||
|
value, err := commandkit.ParseCLIJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if value == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
names, ok := value.([]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("expected a JSON array of collection names or null")
|
||||||
|
}
|
||||||
|
for _, name := range names {
|
||||||
|
if _, ok := name.(string); !ok {
|
||||||
|
return nil, fmt.Errorf("collection names must be strings")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
157
internal/cli/process_command_integration_test.go
Normal file
157
internal/cli/process_command_integration_test.go
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessShowsConfigurationLoadDiagnostics(t *testing.T) {
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, config, finding string
|
||||||
|
}{
|
||||||
|
{"obsolete version", "version: 2\n", "docs/migrations/v3.md"},
|
||||||
|
{"invalid retry budget", "version: 3\nllm: {transient_retries: -1}\n", "retry budget must be non-negative"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(tc.config), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := filepath.Join(dir, "out.json")
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--quiet", "--config", configPath, "--output", output, filepath.Join(dir, "transcript.json")}
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
var loadErr *ck.LoadError
|
||||||
|
if result.Status != 2 || !errors.As(result.Cause, &loadErr) || out.Len() != 0 ||
|
||||||
|
!strings.Contains(stderr.String(), tc.finding) || !strings.Contains(stderr.String(), configPath) {
|
||||||
|
t.Fatalf("status=%d cause=%v out=%q err=%q", result.Status, result.Cause, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("outputs created on load failure: %v, err=%v", entries, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessUsesResolvedReportingMode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"blocked": "file",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
status int
|
||||||
|
stdout bool
|
||||||
|
info bool
|
||||||
|
warning bool
|
||||||
|
}{
|
||||||
|
{"ordinary", nil, 0, true, false, false},
|
||||||
|
{"verbose", []string{"--verbose"}, 0, true, true, false},
|
||||||
|
{"quiet stdout", []string{"--quiet"}, 2, false, false, false},
|
||||||
|
{"quiet file", []string{"--quiet", "--output", filepath.Join(dir, "quiet.json")}, 0, false, false, false},
|
||||||
|
{"verbose after quiet", []string{"--quiet", "--verbose"}, 0, true, true, false},
|
||||||
|
{"quiet after verbose", []string{"--verbose", "--quiet", "--output", filepath.Join(dir, "ordered.json")}, 0, false, false, false},
|
||||||
|
{"debug disabled", []string{"--debug=false", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, false, false},
|
||||||
|
{"debug failure", []string{"--debug", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, true, true},
|
||||||
|
{"debug quiet", []string{"--debug", "--quiet", "--output", filepath.Join(dir, "debug-quiet.json"), "--work-dir", filepath.Join(dir, "blocked")}, 0, false, false, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--config", filepath.Join(dir, "config.yml")}
|
||||||
|
args = append(args, tc.flags...)
|
||||||
|
args = append(args, filepath.Join(dir, "transcript.json"))
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status != tc.status || (out.Len() > 0) != tc.stdout || strings.Contains(stderr.String(), "Completed:") != tc.info || strings.Contains(stderr.String(), "Debug artifact:") != tc.warning {
|
||||||
|
t.Fatalf("status=%d out=%q err=%q", result.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessChecksOnlyTheActiveReportDirectory(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
wantReport bool
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
{"file output ignores blank configured directory", nil, true, false},
|
||||||
|
{"explicit blank directory", []string{"--report-dir="}, false, true},
|
||||||
|
{"disabled reporting ignores explicit blank directory", []string{"--reporting=false", "--report-dir="}, false, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte("version: 3\npipeline: {stages: []}\nreporting: {stdout_dir: ''}\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transcript := filepath.Join(dir, "transcript.json")
|
||||||
|
if err := os.WriteFile(transcript, []byte(`[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := filepath.Join(dir, "output.json")
|
||||||
|
args := append([]string{"process", "--config", configPath, "--output", output}, tc.flags...)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
status := Run(context.Background(), append(args, transcript), &stdout, &stderr)
|
||||||
|
if (status != 0) != tc.wantError || (tc.wantError && !strings.Contains(stderr.String(), "report directory is required")) {
|
||||||
|
t.Fatalf("status=%d stdout=%q stderr=%q", status, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
_, err := os.Stat(filepath.Join(dir, "errors.yml"))
|
||||||
|
if (tc.wantReport && err != nil) || (!tc.wantReport && !os.IsNotExist(err)) {
|
||||||
|
t.Fatalf("report file: %v, want present=%t", err, tc.wantReport)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessCollectionFlags(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\nglossary_collections:\n party: {entries: []}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
value string
|
||||||
|
status int
|
||||||
|
}{{`["party"]`, 0}, {`[]`, 0}, {`null`, 0}, {`[1]`, 2}, {`"party"`, 2}} {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--config", filepath.Join(dir, "config.yml"), "--glossaries", tc.value, filepath.Join(dir, "transcript.json")}
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status != tc.status {
|
||||||
|
t.Fatalf("value=%q status=%d out=%q err=%q", tc.value, outcome.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
160
internal/cli/process_inputs.go
Normal file
160
internal/cli/process_inputs.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
coreio "gitea.maximumdirect.net/eric/audita/v3/internal/core/io"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/preflight"
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processInputs struct {
|
||||||
|
Configuration *ck.Result[config.Config]
|
||||||
|
Runtime *llm.Runtime
|
||||||
|
Source *schema.SourceTranscript
|
||||||
|
RawTranscript []byte
|
||||||
|
ProtectedPaths []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadProcessInputs(ctx context.Context, input processInput, mode commandkit.ReportingMode) (*processInputs, error) {
|
||||||
|
if mode == commandkit.ReportingModeQuiet && input.Output.Value == "" {
|
||||||
|
return nil, fmt.Errorf("quiet processing requires an output file or /dev/null")
|
||||||
|
}
|
||||||
|
selection := ckconfig.Selection{Config: input.Config, Overlay: input.Overlay, Overrides: input.Overrides}
|
||||||
|
loaded, err := loadApplication(ctx, selection, struct{}{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runtime, err := llm.NewRuntime(ctx, loaded, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
raw, err := coreio.ReadRequiredFile(input.Transcript, "transcript")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
protected := append([]string(nil), runtime.SourcePaths...)
|
||||||
|
for _, entry := range loaded.Sources() {
|
||||||
|
if entry.Kind == ck.SourceKindFile && entry.Path != "" {
|
||||||
|
protected = append(protected, entry.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &processInputs{Configuration: loaded, Runtime: runtime, Source: source, RawTranscript: raw, ProtectedPaths: protected}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// preparedProcess groups the output resources and the facts needed for reports.
|
||||||
|
type preparedProcess struct {
|
||||||
|
Prepared *preflight.Prepared
|
||||||
|
RunID string
|
||||||
|
Started time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareProcess(ctx context.Context, input processInput, loaded *processInputs, stdout io.Writer) (*preparedProcess, error) {
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
reportDir := ""
|
||||||
|
reportDirSet := false
|
||||||
|
for _, override := range input.Overrides {
|
||||||
|
if override.Path.String() == "/reporting/stdout_dir" {
|
||||||
|
reportDirSet = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cfg.Reporting.Enabled {
|
||||||
|
reportDir = cfg.Reporting.StdoutDir
|
||||||
|
if input.Output.Value != "" && !reportDirSet {
|
||||||
|
reportDir = filepath.Dir(input.Output.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runID, err := generateRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
started := time.Now().UTC()
|
||||||
|
var secrets []string
|
||||||
|
if cfg.Reporting.Enabled || input.Debug.Value {
|
||||||
|
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
|
||||||
|
}
|
||||||
|
var initial diagnostics.ReportBundle
|
||||||
|
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
|
||||||
|
if cfg.Reporting.Enabled {
|
||||||
|
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
|
||||||
|
for index, item := range cfg.Pipeline.Stages {
|
||||||
|
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
|
||||||
|
}
|
||||||
|
initial, err = renderReportBundle(snapshot, secrets...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
failed = func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle {
|
||||||
|
failedSnapshot := snapshot
|
||||||
|
completed := time.Now().UTC()
|
||||||
|
failedSnapshot.CompletedAt = &completed
|
||||||
|
if primary != nil {
|
||||||
|
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
|
||||||
|
}
|
||||||
|
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var debug *preflight.DebugInput
|
||||||
|
var warnings []error
|
||||||
|
if input.Debug.Value {
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := loaded.Configuration.WriteEffective(&effective); err != nil {
|
||||||
|
warnings = append(warnings, err)
|
||||||
|
}
|
||||||
|
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||||
|
}
|
||||||
|
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
|
||||||
|
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if profile.Backend != nil {
|
||||||
|
_, err = loaded.Runtime.Engine.InspectBackend(ctx, *profile.Backend)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}, Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prepared.Warnings = append(warnings, prepared.Warnings...)
|
||||||
|
return &preparedProcess{Prepared: prepared, RunID: runID, Started: started}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateRunID() (string, error) {
|
||||||
|
var random [12]byte
|
||||||
|
if _, err := rand.Read(random[:]); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "run-" + hex.EncodeToString(random[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) {
|
||||||
|
errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...)
|
||||||
|
if err != nil {
|
||||||
|
return diagnostics.ReportBundle{}, err
|
||||||
|
}
|
||||||
|
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...)
|
||||||
|
if err != nil {
|
||||||
|
return diagnostics.ReportBundle{}, err
|
||||||
|
}
|
||||||
|
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil
|
||||||
|
}
|
||||||
108
internal/cli/process_inputs_test.go
Normal file
108
internal/cli/process_inputs_test.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestQuietDestinationCheckPrecedesInputAcquisition(t *testing.T) {
|
||||||
|
_, err := loadProcessInputs(context.Background(), processInput{Transcript: "missing.json"}, commandkit.ReportingModeQuiet)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "/dev/null") {
|
||||||
|
t.Fatalf("error=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessAcquisitionCapturesConsumedFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"prompts.yml": "prompts:\n - id: extra\n messages:\n - role: user\n content_file: text.txt\n output:\n format: text\n validation_mode: none\n",
|
||||||
|
"text.txt": "example",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
|
||||||
|
result, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"config.yml", "prompts.yml", "text.txt"} {
|
||||||
|
if !slices.Contains(result.ProtectedPaths, filepath.Join(dir, name)) {
|
||||||
|
t.Errorf("missing %s in %v", name, result.ProtectedPaths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareProcessInitializesReportsAndToleratesDebugLoss(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"blocked": "not a directory",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json"), Output: commandkit.Supplied[string]{Value: filepath.Join(dir, "output.json"), Set: true}, Debug: commandkit.Supplied[bool]{Value: true, Set: true}, Overrides: []ck.Override{{Path: ck.Keys("diagnostics", "work_dir"), Value: filepath.Join(dir, "blocked"), Flag: "--work-dir"}}}
|
||||||
|
loaded, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(context.Background(), input, loaded, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
if len(prepared.Prepared.Warnings) == 0 {
|
||||||
|
t.Fatal("debug loss was not reported")
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(dir, "errors.yml"))
|
||||||
|
if err != nil || !strings.Contains(string(report), "in_progress") {
|
||||||
|
t.Fatalf("report=%q err=%v", report, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(input.Output.Value); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("output published early: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionFlagJSON(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
raw string
|
||||||
|
valid bool
|
||||||
|
null bool
|
||||||
|
size int
|
||||||
|
}{
|
||||||
|
{`["party","spells"]`, true, false, 2}, {`[]`, true, false, 0}, {`null`, true, true, 0},
|
||||||
|
{`"party"`, false, false, 0}, {`[1]`, false, false, 0}, {`["x"] true`, false, false, 0},
|
||||||
|
} {
|
||||||
|
result, err := parseCollectionNames(tc.raw)
|
||||||
|
if (err == nil) != tc.valid {
|
||||||
|
t.Fatalf("%s error=%v", tc.raw, err)
|
||||||
|
}
|
||||||
|
if !tc.valid {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if tc.null {
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("null=%v", result)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
list, ok := result.([]any)
|
||||||
|
if !ok || len(list) != tc.size {
|
||||||
|
t.Fatalf("%s result=%v", tc.raw, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
200
internal/cli/process_ordered_integration_test.go
Normal file
200
internal/cli/process_ordered_integration_test.go
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type controlledTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var models []string
|
||||||
|
var prompts []string
|
||||||
|
proposalCalls := 0
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var contents string
|
||||||
|
for _, message := range request.Messages {
|
||||||
|
contents += message.Content
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
models = append(models, request.Model)
|
||||||
|
prompts = append(prompts, contents)
|
||||||
|
if request.Model == "model-one" {
|
||||||
|
proposalCalls++
|
||||||
|
}
|
||||||
|
firstProposalCall := proposalCalls
|
||||||
|
mu.Unlock()
|
||||||
|
result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}`
|
||||||
|
switch request.Model {
|
||||||
|
case "model-one":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}`
|
||||||
|
if firstProposalCall == 1 {
|
||||||
|
result = `{"invalid":true}`
|
||||||
|
}
|
||||||
|
case "model-two":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}`
|
||||||
|
case "model-three":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast","confidence":1}]}`
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(result)
|
||||||
|
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||||
|
})
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": `version: 3
|
||||||
|
glossary_collections:
|
||||||
|
party:
|
||||||
|
entries: [{id: alice, name: Alice, category: person}]
|
||||||
|
spells:
|
||||||
|
entries: [{id: shield, name: Shield, category: spell}]
|
||||||
|
protected_terms: [party, spells]
|
||||||
|
llm: {validation_profile: review}
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
id: "party/../|<tag>"
|
||||||
|
glossaries: [party]
|
||||||
|
llm: {proposal_profile: one}
|
||||||
|
- module: glossary
|
||||||
|
id: spell
|
||||||
|
glossaries: [spells]
|
||||||
|
llm: {proposal_profile: two}
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party, spells]
|
||||||
|
llm: {proposal_profile: three}
|
||||||
|
`,
|
||||||
|
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
loaded.Config.Diagnostics.WorkDir = dir
|
||||||
|
loaded.Config.Reporting.StdoutDir = dir
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
|
||||||
|
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
|
||||||
|
}
|
||||||
|
if len(prompts) > 4 && !strings.Contains(prompts[4], "Alice casts Sheeld.") {
|
||||||
|
t.Fatalf("second proposal missed earlier edit: %s", prompts[4])
|
||||||
|
}
|
||||||
|
if len(result.Snapshot.WorkUnits) != 3 || result.Snapshot.WorkUnits[0].StageID == nil || *result.Snapshot.WorkUnits[0].StageID != "party/../|<tag>" || result.Snapshot.WorkUnits[1].StageIndex != 1 || result.Snapshot.WorkUnits[2].StageID != nil {
|
||||||
|
t.Fatalf("work unit identities: %+v", result.Snapshot.WorkUnits)
|
||||||
|
}
|
||||||
|
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
||||||
|
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
||||||
|
}
|
||||||
|
markdown := string(reporting.Markdown(result.Snapshot))
|
||||||
|
if !strings.Contains(markdown, `1: glossary (party/../\|<tag>)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
|
||||||
|
t.Fatalf("report labels: %s", markdown)
|
||||||
|
}
|
||||||
|
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
|
||||||
|
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatal(readErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) {
|
||||||
|
t.Fatalf("%s missing attribution: %s", artifact, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||||
|
t.Fatalf("models=%v", models)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
|
||||||
|
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
|
||||||
|
})
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
loaded.Config.Reporting.StdoutDir = dir
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" || result.Snapshot.Errors[0].StageID == nil || *result.Snapshot.Errors[0].StageID != "failing" {
|
||||||
|
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
113
internal/cli/process_output.go
Normal file
113
internal/cli/process_output.go
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// finishProcess delivers the transcript and reports before emitting success information.
|
||||||
|
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
|
||||||
|
reporter := inv.Reporter()
|
||||||
|
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
|
||||||
|
safe := redaction.New(secrets)
|
||||||
|
for _, warning := range prepared.Prepared.Warnings {
|
||||||
|
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
|
||||||
|
}
|
||||||
|
completed := time.Now().UTC()
|
||||||
|
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
|
||||||
|
snapshot.TranscriptPath, snapshot.OutputPath = input.Transcript, input.Output.Value
|
||||||
|
if primary == nil {
|
||||||
|
primary = inv.Context.Err()
|
||||||
|
}
|
||||||
|
snapshot.Status = reporting.StatusCompleted
|
||||||
|
if primary != nil {
|
||||||
|
snapshot = failedProcessingSnapshot(snapshot, primary, inv.Context.Err())
|
||||||
|
}
|
||||||
|
reportPaths := prepared.Prepared.Output.ReportPaths()
|
||||||
|
var reports diagnostics.ReportBundle
|
||||||
|
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
|
||||||
|
if len(reportPaths) > 0 {
|
||||||
|
var err error
|
||||||
|
reports, err = renderReportBundle(snapshot, secrets...)
|
||||||
|
if err != nil {
|
||||||
|
if primary == nil {
|
||||||
|
primary = err
|
||||||
|
}
|
||||||
|
snapshot.Status = reporting.StatusFailed
|
||||||
|
}
|
||||||
|
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
|
||||||
|
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
|
||||||
|
if bundleErr != nil {
|
||||||
|
return reports
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if canceled := inv.Context.Err(); primary == nil && canceled != nil {
|
||||||
|
primary = canceled
|
||||||
|
snapshot = failedProcessingSnapshot(snapshot, canceled, canceled)
|
||||||
|
if failed != nil {
|
||||||
|
reports = failed(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
|
||||||
|
if result != nil {
|
||||||
|
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
|
||||||
|
}
|
||||||
|
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
|
||||||
|
for _, path := range reportPaths {
|
||||||
|
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))
|
||||||
|
}
|
||||||
|
if prepared.Prepared.Debug != nil {
|
||||||
|
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path())))
|
||||||
|
}
|
||||||
|
return commandkit.Success()
|
||||||
|
}
|
||||||
|
|
||||||
|
func failedProcessingSnapshot(snapshot reporting.Snapshot, primary, contextErr error) reporting.Snapshot {
|
||||||
|
for _, record := range snapshot.Errors {
|
||||||
|
if record.Scope == "run" {
|
||||||
|
return failedOutputSnapshot(snapshot, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record := reporting.ErrorRecord{Scope: "run", Reason: "processing_failure", Message: "Processing failed"}
|
||||||
|
if errors.Is(primary, context.Canceled) || (contextErr != nil && errors.Is(primary, contextErr)) {
|
||||||
|
record.Phase = "processing"
|
||||||
|
record.Reason = string(outcomes.ReasonCanceled)
|
||||||
|
record.Message = "Processing was canceled."
|
||||||
|
}
|
||||||
|
snapshot.Errors = append(snapshot.Errors, record)
|
||||||
|
return failedOutputSnapshot(snapshot, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedOutputSnapshot derives the count from the same run records sent to each report.
|
||||||
|
func failedOutputSnapshot(snapshot reporting.Snapshot, failures []diagnostics.FileFailure) reporting.Snapshot {
|
||||||
|
snapshot.Status = reporting.StatusFailed
|
||||||
|
if len(failures) > 0 {
|
||||||
|
snapshot.Errors = append([]reporting.ErrorRecord(nil), snapshot.Errors...)
|
||||||
|
}
|
||||||
|
for _, failure := range failures {
|
||||||
|
record := reporting.ErrorRecord{Scope: "run", Reason: "output_failure", Message: "Required output delivery failed", OutputPath: failure.Path}
|
||||||
|
if failure.Kind == "report initialization" {
|
||||||
|
record.Phase = "initialization"
|
||||||
|
record.Message = "Required report initialization failed"
|
||||||
|
}
|
||||||
|
snapshot.Errors = append(snapshot.Errors, record)
|
||||||
|
}
|
||||||
|
count := 0
|
||||||
|
for _, record := range snapshot.Errors {
|
||||||
|
if record.Scope == "run" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snapshot.Summary.RunErrors = &count
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
197
internal/cli/process_output_test.go
Normal file
197
internal/cli/process_output_test.go
Normal file
@@ -0,0 +1,197 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedDeliveryFollowsReportingMode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
file bool
|
||||||
|
wantInfo bool
|
||||||
|
}{
|
||||||
|
{"ordinary", nil, false, false}, {"verbose", []string{"--verbose"}, false, true}, {"quiet file", []string{"--quiet"}, true, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
|
||||||
|
if tc.file {
|
||||||
|
input.Output = commandkit.Supplied[string]{Value: filepath.Join(dir, "result.json"), Set: true}
|
||||||
|
}
|
||||||
|
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||||
|
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
|
||||||
|
if err != nil {
|
||||||
|
return commandkit.Usage(err.Error())
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||||
|
if err != nil {
|
||||||
|
return commandkit.Failure(err.Error())
|
||||||
|
}
|
||||||
|
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||||
|
}})
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := append([]string{"deliver"}, tc.flags...)
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
t.Fatalf("outcome=%+v err=%s", outcome, stderr.String())
|
||||||
|
}
|
||||||
|
if tc.file {
|
||||||
|
data, err := os.ReadFile(input.Output.Value)
|
||||||
|
if err != nil || string(data) != "transcript bytes" {
|
||||||
|
t.Fatalf("file=%q err=%v", data, err)
|
||||||
|
}
|
||||||
|
if out.Len() != 0 {
|
||||||
|
t.Fatalf("stdout=%q", out.String())
|
||||||
|
}
|
||||||
|
} else if out.String() != "transcript bytes" {
|
||||||
|
t.Fatalf("stdout=%q", out.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(stderr.String(), "Completed:") != tc.wantInfo {
|
||||||
|
t.Fatalf("stderr=%q", stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Chdir(dir)
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
|
||||||
|
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||||
|
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
|
||||||
|
if err != nil {
|
||||||
|
return commandkit.Failure(err.Error())
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||||
|
if err != nil {
|
||||||
|
return commandkit.Failure(err.Error())
|
||||||
|
}
|
||||||
|
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||||
|
}})
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"deliver", "--verbose"}, commandkit.Streams{Out: shortProcessWriter{}, Err: &stderr})
|
||||||
|
if outcome.Status == 0 || strings.Contains(stderr.String(), "Completed:") {
|
||||||
|
t.Fatalf("status=%d stderr=%q", outcome.Status, stderr.String())
|
||||||
|
}
|
||||||
|
var errs reporting.ErrorsDocument
|
||||||
|
var recoveries reporting.RecoveriesDocument
|
||||||
|
for path, target := range map[string]any{"errors.yml": &errs, "recoveries.yml": &recoveries} {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := yaml.Unmarshal(data, target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errs.Status != reporting.StatusFailed || recoveries.Status != reporting.StatusFailed ||
|
||||||
|
errs.Summary.RunErrors == nil || *errs.Summary.RunErrors != 1 ||
|
||||||
|
recoveries.Summary.RunErrors == nil || *recoveries.Summary.RunErrors != 1 ||
|
||||||
|
len(errs.Errors) != 1 || errs.Errors[0].Reason != "output_failure" {
|
||||||
|
t.Fatalf("failed output reports disagree: %+v %+v", errs, recoveries)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline: {stages: []}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"output.json": "previous transcript",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{
|
||||||
|
Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true},
|
||||||
|
Transcript: filepath.Join(dir, "transcript.json"),
|
||||||
|
Output: commandkit.Supplied[string]{Value: filepath.Join(dir, "output.json"), Set: true},
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
|
||||||
|
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(inv.Context, loaded, prepared, sourceToCanonicalTranscript(loaded.Source), func(error) {})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil)
|
||||||
|
}})
|
||||||
|
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
outcome := app.Execute(ctx, []string{"deliver", "--verbose"}, commandkit.Streams{Out: &stdout, Err: &stderr})
|
||||||
|
if outcome.Status == 0 || !errors.Is(outcome.Cause, context.Canceled) || strings.Contains(stderr.String(), "Completed:") {
|
||||||
|
t.Fatalf("outcome=%+v stderr=%q", outcome, stderr.String())
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(input.Output.Value); err != nil || string(data) != "previous transcript" {
|
||||||
|
t.Fatalf("transcript changed after cancellation: %q, %v", data, err)
|
||||||
|
}
|
||||||
|
var errorsDoc reporting.ErrorsDocument
|
||||||
|
var recoveries reporting.RecoveriesDocument
|
||||||
|
for name, target := range map[string]any{"errors.yml": &errorsDoc, "recoveries.yml": &recoveries} {
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := yaml.Unmarshal(data, target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errorsDoc.Status != reporting.StatusFailed || recoveries.Status != reporting.StatusFailed ||
|
||||||
|
len(errorsDoc.Errors) != 1 || errorsDoc.Errors[0].Reason != "canceled" {
|
||||||
|
t.Fatalf("reports did not capture cancellation: %+v %+v", errorsDoc, recoveries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type shortProcessWriter struct{}
|
||||||
|
|
||||||
|
func (shortProcessWriter) Write(data []byte) (int, error) { return len(data) / 2, nil }
|
||||||
81
internal/cli/process_pipeline.go
Normal file
81
internal/cli/process_pipeline.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
|
)
|
||||||
|
|
||||||
|
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
specs := contracts.ResolveModuleRunSpecs(cfg)
|
||||||
|
debug := prepared.Prepared.Debug
|
||||||
|
var observer *llm.ObservationCollector
|
||||||
|
var hooks llm.CallHooks
|
||||||
|
if debug != nil {
|
||||||
|
observer = llm.NewObservationCollector()
|
||||||
|
hooks = observer.Hooks()
|
||||||
|
}
|
||||||
|
for i := range specs {
|
||||||
|
item := &specs[i]
|
||||||
|
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, hooks)
|
||||||
|
if err != nil {
|
||||||
|
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
|
||||||
|
}
|
||||||
|
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, hooks)
|
||||||
|
if err != nil {
|
||||||
|
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
|
||||||
|
}
|
||||||
|
item.ProposalClient, item.ValidationClient = proposal, validation
|
||||||
|
if item.Settings.LLM.CorrectionRepairAttempts > 0 {
|
||||||
|
repair, err := loaded.Runtime.NewRoleClient("correction_repair", *item.Settings.LLM.CorrectionRepairProfile, item.Settings.LLM, hooks)
|
||||||
|
if err != nil {
|
||||||
|
return processResult{}, fmt.Errorf("%s correction repair client: %w", item.InstanceName, err)
|
||||||
|
}
|
||||||
|
item.RepairClient = repair
|
||||||
|
item.CaptureProposalExchange = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
debugPath := prepared.DebugPath()
|
||||||
|
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||||
|
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||||
|
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
|
||||||
|
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
|
||||||
|
})
|
||||||
|
if debug != nil {
|
||||||
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
|
||||||
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||||
|
}
|
||||||
|
var failures []processreport.RunFailure
|
||||||
|
if runErr != nil {
|
||||||
|
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
|
||||||
|
var cause *structuredoutput.Failure
|
||||||
|
if errors.As(runErr, &cause) {
|
||||||
|
failure.Reason = cause.Reason
|
||||||
|
} else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) {
|
||||||
|
failure.Reason = outcomes.ReasonCanceled
|
||||||
|
}
|
||||||
|
if len(output.ModuleResults) > 0 {
|
||||||
|
last := output.ModuleResults[len(output.ModuleResults)-1]
|
||||||
|
if last.Status == runner.ModuleStatusFailed {
|
||||||
|
index := last.StageIndex
|
||||||
|
failure.Origin = &outcomes.Reference{StageIndex: &index, StageID: last.StageID, ModuleKey: last.ModuleKey, ModuleInstance: last.ModuleInstance}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failures = append(failures, failure)
|
||||||
|
}
|
||||||
|
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
|
||||||
|
ModuleSpecs: specs, RunOutput: &output, RunFailures: failures, SummaryOnly: !cfg.Reporting.Enabled})
|
||||||
|
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
||||||
|
}
|
||||||
@@ -2,13 +2,11 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"flag"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ReleaseKit/buildinfo"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
||||||
@@ -30,241 +28,12 @@ func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transc
|
|||||||
return &schema.Transcript{Segments: segments}
|
return &schema.Transcript{Segments: segments}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run executes the Audita CLI with the provided arguments and streams.
|
// Run executes the application with the provided arguments and streams.
|
||||||
func Run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
func Run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
if len(args) == 0 {
|
app, err := buildApplication()
|
||||||
writeRootUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if len(args) == 2 && args[0] == "help" {
|
|
||||||
switch args[1] {
|
|
||||||
case "process":
|
|
||||||
writeProcessUsage(stdout)
|
|
||||||
return 0
|
|
||||||
case "config":
|
|
||||||
writeConfigUsage(stdout)
|
|
||||||
return 0
|
|
||||||
case "version":
|
|
||||||
writeVersionUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if isHelpCommand(args) {
|
|
||||||
writeRootUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if args[0] == "process" {
|
|
||||||
return runProcess(ctx, args[1:], stdout, stderr)
|
|
||||||
}
|
|
||||||
if args[0] == "config" {
|
|
||||||
return runConfig(ctx, args[1:], stdout, stderr)
|
|
||||||
}
|
|
||||||
if args[0] == "version" {
|
|
||||||
return runVersion(args[1:], stdout, stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
|
|
||||||
writeRootUsage(stderr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
func runVersion(args []string, stdout, stderr io.Writer) int {
|
|
||||||
flags := flag.NewFlagSet("version", flag.ContinueOnError)
|
|
||||||
flags.SetOutput(stderr)
|
|
||||||
flags.Usage = func() {} // Route usage after parsing so help stays on stdout.
|
|
||||||
if err := flags.Parse(args); err != nil {
|
|
||||||
if err == flag.ErrHelp {
|
|
||||||
writeVersionUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
writeVersionUsage(stderr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if positional := flags.Args(); len(positional) > 0 {
|
|
||||||
fmt.Fprintf(stderr, "audita version: unexpected argument %q\n\n", positional[0])
|
|
||||||
writeVersionUsage(stderr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
fmt.Fprintf(stdout, "audita %s\n", buildinfo.Version())
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func runConfig(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|
||||||
if len(args) == 0 || isHelpCommand(args) || args[0] == "--help" || args[0] == "-h" {
|
|
||||||
writeConfigUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
switch args[0] {
|
|
||||||
case "validate":
|
|
||||||
return runConfigValidate(ctx, args[1:], stdout, stderr)
|
|
||||||
case "print-effective":
|
|
||||||
return runConfigPrintEffective(ctx, args[1:], stdout, stderr)
|
|
||||||
default:
|
|
||||||
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
|
|
||||||
writeConfigUsage(stderr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|
||||||
input, help, err := parseConfigInput("validate", args, stderr)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "audita config validate: %s\n", configInputError(err))
|
fmt.Fprintln(stderr, err)
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if help {
|
|
||||||
writeConfigValidateUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, "validate")
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config validate: %s\n", configInputError(err))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
|
|
||||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
return 2
|
return app.Execute(ctx, args, commandkit.Streams{Out: stdout, Err: stderr}).Status
|
||||||
}
|
|
||||||
fmt.Fprintln(stdout, "config is valid")
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|
||||||
input, help, err := parseConfigInput("print-effective", args, stderr)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %s\n", configInputError(err))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if help {
|
|
||||||
writeConfigPrintEffectiveUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, "print-effective")
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %s\n", configInputError(err))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
|
|
||||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if input.View == "settings" {
|
|
||||||
err = selected.Result.WriteEffective(stdout)
|
|
||||||
} else {
|
|
||||||
var output []byte
|
|
||||||
if input.View == "profiles" {
|
|
||||||
output, err = llmRuntime.SelectedProfilesYAML()
|
|
||||||
} else {
|
|
||||||
output, err = llm.ExplanationYAML(selected.Result)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = stdout.Write(output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func isHelpCommand(args []string) bool {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if len(args) == 1 {
|
|
||||||
switch args[0] {
|
|
||||||
case "help", "-h", "--help":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(args) == 2 && args[0] == "help" {
|
|
||||||
switch args[1] {
|
|
||||||
case "process":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeRootUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita <command> [options]")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Commands:")
|
|
||||||
fmt.Fprintln(w, " process Process a transcript JSON file")
|
|
||||||
fmt.Fprintln(w, " config Validate and inspect config")
|
|
||||||
fmt.Fprintln(w, " version Print the Audita version")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Example:")
|
|
||||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeVersionUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Print the Audita version.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita version")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Validate and inspect Audita config.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita config <command> [flags]")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Commands:")
|
|
||||||
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
|
|
||||||
fmt.Fprintln(w, " print-effective Print redacted effective settings YAML")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigValidateUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Validate a versioned YAML config file.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita config validate --config <path> [--overlay <name>] [setting flags]")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigPrintEffectiveUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Print redacted effective settings YAML.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita config print-effective [--config <path>] [--overlay <name>] [setting flags]")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeProcessUsage(w io.Writer) {
|
|
||||||
fmt.Fprintln(w, "Process a transcript JSON file.")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Usage:")
|
|
||||||
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Flags:")
|
|
||||||
for _, name := range []string{
|
|
||||||
"config", "overlay", "glossary", "output",
|
|
||||||
"proposal-profile", "validation-profile", "modules", "output-schema",
|
|
||||||
"validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens",
|
|
||||||
"normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "normalize-max-segment-tokens",
|
|
||||||
"glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold",
|
|
||||||
"transcript-description", "work-dir",
|
|
||||||
} {
|
|
||||||
fmt.Fprintf(w, " --%s\n", name)
|
|
||||||
}
|
|
||||||
fmt.Fprintln(w, " --reporting[=true|false] Write the three-file report bundle (default true)")
|
|
||||||
fmt.Fprintln(w, " --report-dir DIR Select the report directory (stdout setting for stdout; output parent by default for files)")
|
|
||||||
fmt.Fprintln(w, " --debug[=true|false] Persist detailed diagnostics (default false)")
|
|
||||||
fmt.Fprintln(w)
|
|
||||||
fmt.Fprintln(w, "Example:")
|
|
||||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,65 +12,42 @@ import (
|
|||||||
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
profiles := filepath.Join(dir, "profiles")
|
|
||||||
if err := os.Mkdir(profiles, 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
main := filepath.Join(dir, "config.yml")
|
main := filepath.Join(dir, "config.yml")
|
||||||
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
|
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": operatorDocument} {
|
||||||
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
return main
|
return main
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
||||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||||
definitions := map[string]string{
|
definitions := map[string]string{
|
||||||
"malformed operator YAML": "profiles:\n proposal: [\n",
|
"malformed operator YAML": "profiles: [\n",
|
||||||
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
|
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
|
||||||
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
|
"missing operator message resource": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content_file: missing-instruction.md\n output:\n format: text\n validation_mode: none\n",
|
||||||
}
|
}
|
||||||
commands := []struct {
|
for name, document := range definitions {
|
||||||
name string
|
t.Run(name, func(t *testing.T) {
|
||||||
args []string
|
|
||||||
}{
|
|
||||||
{name: "validate", args: []string{"config", "validate"}},
|
|
||||||
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
|
|
||||||
}
|
|
||||||
for definition, document := range definitions {
|
|
||||||
for _, command := range commands {
|
|
||||||
t.Run(definition+"/"+command.name, func(t *testing.T) {
|
|
||||||
main := writeRuntimeErrorConfig(t, document)
|
main := writeRuntimeErrorConfig(t, document)
|
||||||
args := append(append([]string(nil), command.args...), "--config", main)
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
|
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
|
||||||
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
|
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
|
||||||
}
|
}
|
||||||
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
|
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
|
||||||
t.Fatalf("configured secret leaked: %q", stderr.String())
|
t.Fatal("configured secret leaked")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
|
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
|
||||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
|
||||||
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
|
|
||||||
commands := [][]string{
|
|
||||||
{"config", "validate", "--config", main},
|
|
||||||
{"config", "print-effective", "--config", main, "--view", "profiles"},
|
|
||||||
}
|
|
||||||
for _, args := range commands {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
|
if code := Run(ctx, []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
|
||||||
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
|
t.Fatalf("cancellation succeeded: stdout=%q stderr=%q", stdout.String(), stderr.String())
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ReleaseKit/buildinfo"
|
"go.maximumdirect.net/releasekit/buildinfo"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestVersionCommandRunsWithoutApplicationSetup(t *testing.T) {
|
func TestVersionCommandRunsWithoutApplicationSetup(t *testing.T) {
|
||||||
@@ -30,12 +30,10 @@ func TestVersionCommandHelp(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "root help", args: []string{"--help"}, want: "version Print the Audita version"},
|
{name: "root help", args: []string{"--help"}, want: "version Show application or CommandKit version."},
|
||||||
{name: "help version", args: []string{"help", "version"}, want: "audita version"},
|
{name: "help version", args: []string{"help", "version"}, want: "audita version"},
|
||||||
{name: "short flag", args: []string{"version", "-h"}, want: "audita version"},
|
{name: "short flag", args: []string{"version", "-h"}, want: "audita version"},
|
||||||
{name: "long flag", args: []string{"version", "--help"}, want: "audita version"},
|
{name: "long flag", args: []string{"version", "--help"}, want: "audita version"},
|
||||||
{name: "parser help", args: []string{"version", "--h"}, want: "audita version"},
|
|
||||||
{name: "help short circuits", args: []string{"version", "-h", "--help", "extra"}, want: "audita version"},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
package chunking
|
|
||||||
|
|
||||||
// Summary provides a concise overview of chunking results for reports
|
|
||||||
type Summary struct {
|
|
||||||
ChunkCount int `json:"chunk_count"`
|
|
||||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
|
||||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
|
||||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
|
||||||
TargetSections *int `json:"target_sections,omitempty"`
|
|
||||||
MaxSectionTokens int `json:"max_section_tokens"`
|
|
||||||
MinSectionTokens int `json:"min_section_tokens"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
|
||||||
type ChunkSummary struct {
|
|
||||||
Index int `json:"index"`
|
|
||||||
StartSegmentID int `json:"start_segment_id"`
|
|
||||||
EndSegmentID int `json:"end_segment_id"`
|
|
||||||
EstimatedTokens int `json:"estimated_tokens"`
|
|
||||||
SegmentCount int `json:"segment_count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// DetailedSummary provides per-chunk details for diagnostics
|
|
||||||
type DetailedSummary struct {
|
|
||||||
Summary `json:",inline"`
|
|
||||||
Chunks []ChunkSummary `json:"chunks"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ComputeSummary creates a Summary from sections and config
|
|
||||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
|
||||||
if len(sections) == 0 {
|
|
||||||
return Summary{
|
|
||||||
ChunkCount: 0,
|
|
||||||
MaxSectionTokens: config.MaxSectionTokens,
|
|
||||||
MinSectionTokens: config.MinSectionTokens,
|
|
||||||
TargetSections: config.TargetSections,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
minTokens := sections[0].EstimatedTokens
|
|
||||||
maxTokens := sections[0].EstimatedTokens
|
|
||||||
totalTokens := 0
|
|
||||||
|
|
||||||
for _, sec := range sections {
|
|
||||||
if sec.EstimatedTokens < minTokens {
|
|
||||||
minTokens = sec.EstimatedTokens
|
|
||||||
}
|
|
||||||
if sec.EstimatedTokens > maxTokens {
|
|
||||||
maxTokens = sec.EstimatedTokens
|
|
||||||
}
|
|
||||||
totalTokens += sec.EstimatedTokens
|
|
||||||
}
|
|
||||||
|
|
||||||
return Summary{
|
|
||||||
ChunkCount: len(sections),
|
|
||||||
MinEstimatedTokens: minTokens,
|
|
||||||
MaxEstimatedTokens: maxTokens,
|
|
||||||
TotalEstimatedTokens: totalTokens,
|
|
||||||
TargetSections: config.TargetSections,
|
|
||||||
MaxSectionTokens: config.MaxSectionTokens,
|
|
||||||
MinSectionTokens: config.MinSectionTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
|
||||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
|
||||||
summary := ComputeSummary(sections, config)
|
|
||||||
|
|
||||||
chunks := make([]ChunkSummary, len(sections))
|
|
||||||
for i, sec := range sections {
|
|
||||||
chunks[i] = ChunkSummary{
|
|
||||||
Index: sec.Index,
|
|
||||||
StartSegmentID: sec.StartSegmentID,
|
|
||||||
EndSegmentID: sec.EndSegmentID,
|
|
||||||
EstimatedTokens: sec.EstimatedTokens,
|
|
||||||
SegmentCount: len(sec.Segments),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return DetailedSummary{
|
|
||||||
Summary: summary,
|
|
||||||
Chunks: chunks,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package chunking
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestComputeSummary(t *testing.T) {
|
|
||||||
config := ChunkingConfig{
|
|
||||||
MaxSectionTokens: 100,
|
|
||||||
MinSectionTokens: 10,
|
|
||||||
}
|
|
||||||
|
|
||||||
sections := []Section{
|
|
||||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
|
||||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
|
||||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := ComputeSummary(sections, config)
|
|
||||||
|
|
||||||
if summary.ChunkCount != 3 {
|
|
||||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
|
||||||
}
|
|
||||||
if summary.MinEstimatedTokens != 20 {
|
|
||||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
|
||||||
}
|
|
||||||
if summary.MaxEstimatedTokens != 50 {
|
|
||||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
|
||||||
}
|
|
||||||
if summary.TotalEstimatedTokens != 100 {
|
|
||||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
|
||||||
}
|
|
||||||
if summary.MaxSectionTokens != 100 {
|
|
||||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
|
||||||
}
|
|
||||||
if summary.MinSectionTokens != 10 {
|
|
||||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
|
||||||
}
|
|
||||||
if summary.TargetSections != nil {
|
|
||||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
|
||||||
target := 5
|
|
||||||
config := ChunkingConfig{
|
|
||||||
MaxSectionTokens: 100,
|
|
||||||
MinSectionTokens: 10,
|
|
||||||
TargetSections: &target,
|
|
||||||
}
|
|
||||||
|
|
||||||
sections := []Section{
|
|
||||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := ComputeSummary(sections, config)
|
|
||||||
|
|
||||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
|
||||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
|
||||||
config := ChunkingConfig{
|
|
||||||
MaxSectionTokens: 100,
|
|
||||||
MinSectionTokens: 10,
|
|
||||||
}
|
|
||||||
|
|
||||||
sections := []Section{}
|
|
||||||
|
|
||||||
summary := ComputeSummary(sections, config)
|
|
||||||
|
|
||||||
if summary.ChunkCount != 0 {
|
|
||||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
|
||||||
}
|
|
||||||
if summary.MinEstimatedTokens != 0 {
|
|
||||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
|
||||||
}
|
|
||||||
if summary.MaxSectionTokens != 100 {
|
|
||||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestComputeDetailedSummary(t *testing.T) {
|
|
||||||
config := ChunkingConfig{
|
|
||||||
MaxSectionTokens: 100,
|
|
||||||
MinSectionTokens: 10,
|
|
||||||
}
|
|
||||||
|
|
||||||
sections := []Section{
|
|
||||||
{
|
|
||||||
Index: 0,
|
|
||||||
EstimatedTokens: 30,
|
|
||||||
StartSegmentID: 1,
|
|
||||||
EndSegmentID: 2,
|
|
||||||
Segments: make([]schema.Segment, 2), // 2 segments
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Index: 1,
|
|
||||||
EstimatedTokens: 50,
|
|
||||||
StartSegmentID: 3,
|
|
||||||
EndSegmentID: 5,
|
|
||||||
Segments: make([]schema.Segment, 3), // 3 segments
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
detailed := ComputeDetailedSummary(sections, config)
|
|
||||||
|
|
||||||
if detailed.ChunkCount != 2 {
|
|
||||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
|
||||||
}
|
|
||||||
if len(detailed.Chunks) != 2 {
|
|
||||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check first chunk
|
|
||||||
if detailed.Chunks[0].Index != 0 {
|
|
||||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
|
||||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
|
||||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
|
||||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[0].SegmentCount != 2 {
|
|
||||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check second chunk
|
|
||||||
if detailed.Chunks[1].Index != 1 {
|
|
||||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
|
||||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
|
||||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
|
||||||
}
|
|
||||||
if detailed.Chunks[1].SegmentCount != 3 {
|
|
||||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package chunking
|
package chunking
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
"unicode"
|
"unicode"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -24,24 +23,24 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
|||||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||||
// The estimate is deterministic and stable for the same input text.
|
// The estimate is deterministic and stable for the same input text.
|
||||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||||
if text == "" {
|
words, punctuation := e.CountComponents(text)
|
||||||
return 0
|
return words + punctuation/2
|
||||||
}
|
}
|
||||||
|
|
||||||
// Simple heuristic: split on whitespace and count non-empty segments
|
// CountComponents returns unrounded counts for budgets composed from fragments.
|
||||||
words := strings.Fields(text)
|
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
|
||||||
tokenCount := len(words)
|
inWord := false
|
||||||
|
|
||||||
// Add some estimate for punctuation that might be separate tokens
|
|
||||||
punctuationCount := 0
|
|
||||||
for _, r := range text {
|
for _, r := range text {
|
||||||
|
nonspace := !unicode.IsSpace(r)
|
||||||
|
if nonspace && !inWord {
|
||||||
|
words++
|
||||||
|
}
|
||||||
|
inWord = nonspace
|
||||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||||
punctuationCount++
|
punctuation++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return words, punctuation
|
||||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
|
||||||
return tokenCount + (punctuationCount / 2)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||||
|
|||||||
@@ -5,22 +5,22 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
|
|
||||||
DefaultTranscriptDescriptionMaxChars = 500
|
DefaultTranscriptDescriptionMaxChars = 500
|
||||||
DefaultValidationMaxPromptTokens = 2048
|
DefaultValidationMaxPromptTokens = 2048
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config is the version-2 application configuration. Invocation paths are not settings.
|
// Config contains application settings. Invocation paths are not settings.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Version int `yaml:"version"`
|
Version int `yaml:"version"`
|
||||||
ProfilesDir string `yaml:"profiles_dir"`
|
Glossaries []string `yaml:"glossaries"`
|
||||||
Backends map[string]Backend `yaml:"backends"`
|
ProtectedTerms []string `yaml:"protected_terms"`
|
||||||
|
GlossaryCollections map[string]GlossaryCollection `yaml:"glossary_collections"`
|
||||||
LLM LLM `yaml:"llm"`
|
LLM LLM `yaml:"llm"`
|
||||||
Pipeline Pipeline `yaml:"pipeline"`
|
Pipeline Pipeline `yaml:"pipeline"`
|
||||||
Output Output `yaml:"output"`
|
Output Output `yaml:"output"`
|
||||||
@@ -33,21 +33,39 @@ type Config struct {
|
|||||||
Diagnostics Diagnostics `yaml:"diagnostics"`
|
Diagnostics Diagnostics `yaml:"diagnostics"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Backend struct {
|
type GlossaryCollection struct {
|
||||||
Endpoint string `yaml:"endpoint"`
|
Entries []GlossaryEntry `yaml:"entries"`
|
||||||
Concurrency int `yaml:"concurrency"`
|
}
|
||||||
QueueCapacity int `yaml:"queue_capacity"`
|
type GlossaryEntry struct {
|
||||||
Credential *ck.Secret `yaml:"credential"`
|
ID string `yaml:"id"`
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Category string `yaml:"category"`
|
||||||
|
Description string `yaml:"description"`
|
||||||
|
Aliases []string `yaml:"aliases"`
|
||||||
|
Plural *string `yaml:"plural"`
|
||||||
|
Extra map[string]any `yaml:",inline"`
|
||||||
}
|
}
|
||||||
type LLM struct {
|
type LLM struct {
|
||||||
ProposalProfile string `yaml:"proposal_profile"`
|
ProposalProfile *string `yaml:"proposal_profile"`
|
||||||
ValidationProfile string `yaml:"validation_profile"`
|
ValidationProfile *string `yaml:"validation_profile"`
|
||||||
|
CorrectionRepairProfile *string `yaml:"correction_repair_profile"`
|
||||||
|
CorrectionRepairAttempts int `yaml:"correction_repair_attempts"`
|
||||||
TransientRetries int `yaml:"transient_retries"`
|
TransientRetries int `yaml:"transient_retries"`
|
||||||
RepairAttempts int `yaml:"repair_attempts"`
|
RepairAttempts int `yaml:"repair_attempts"`
|
||||||
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
||||||
}
|
}
|
||||||
type Pipeline struct {
|
type Pipeline struct {
|
||||||
Modules []string `yaml:"modules"`
|
Stages []Stage `yaml:"stages"`
|
||||||
|
}
|
||||||
|
type Stage struct {
|
||||||
|
Module string `yaml:"module"`
|
||||||
|
ID *string `yaml:"id"`
|
||||||
|
Glossaries []string `yaml:"glossaries"`
|
||||||
|
ProtectedTerms []string `yaml:"protected_terms"`
|
||||||
|
ConfidenceThreshold float64 `yaml:"confidence_threshold"`
|
||||||
|
LLM LLM `yaml:"llm"`
|
||||||
|
Validation Validation `yaml:"validation"`
|
||||||
|
Context Context `yaml:"context"`
|
||||||
}
|
}
|
||||||
type Output struct {
|
type Output struct {
|
||||||
Schema string `yaml:"schema"`
|
Schema string `yaml:"schema"`
|
||||||
@@ -76,7 +94,6 @@ type Context struct {
|
|||||||
Description string `yaml:"description"`
|
Description string `yaml:"description"`
|
||||||
}
|
}
|
||||||
type Diagnostics struct {
|
type Diagnostics struct {
|
||||||
Debug bool `yaml:"debug"`
|
|
||||||
WorkDir string `yaml:"work_dir"`
|
WorkDir string `yaml:"work_dir"`
|
||||||
}
|
}
|
||||||
type Reporting struct {
|
type Reporting struct {
|
||||||
@@ -86,9 +103,9 @@ type Reporting struct {
|
|||||||
|
|
||||||
func defaults() Config {
|
func defaults() Config {
|
||||||
return Config{
|
return Config{
|
||||||
ProfilesDir: "profiles",
|
Version: 3,
|
||||||
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
|
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, CorrectionRepairAttempts: 1, OperationTimeout: 15 * time.Minute},
|
||||||
Pipeline: Pipeline{Modules: []string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}},
|
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
|
||||||
Output: Output{Schema: "bare-segments"},
|
Output: Output{Schema: "bare-segments"},
|
||||||
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
|
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
|
||||||
Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048},
|
Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048},
|
||||||
@@ -99,38 +116,34 @@ func defaults() Config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load resolves the selected main file, optional overlay, and explicit setting overrides.
|
const glossaryImports = `config:
|
||||||
// Its result retains safe effective output and provenance for CLI composition.
|
imports:
|
||||||
func Load(source ck.Source, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
- path: glossary.yml
|
||||||
return ck.Load(source, ck.Options[Config]{
|
required: false
|
||||||
|
destination: [glossary_collections]
|
||||||
|
- dir: glossary
|
||||||
|
required: false
|
||||||
|
recursive: true
|
||||||
|
pattern: ["*.yml", "*.yaml"]
|
||||||
|
destination: [glossary_collections]
|
||||||
|
`
|
||||||
|
|
||||||
|
// Load resolves the selected main source and preserves native source evidence.
|
||||||
|
func Load(input ck.Input, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
||||||
|
input.Imports = true
|
||||||
|
return ck.LoadLayered(input, ck.LayeredOptions[Config]{
|
||||||
Overlay: overlay, Overrides: overrides,
|
Overlay: overlay, Overrides: overrides,
|
||||||
|
Underlays: []ck.Underlay{{Inputs: []ck.Input{{Source: ck.Bytes("Audita glossary imports", []byte(glossaryImports)), Imports: true, MainRelative: ck.MainRelative()}}}},
|
||||||
Definition: ck.Definition[Config]{
|
Definition: ck.Definition[Config]{
|
||||||
Defaults: defaults,
|
Defaults: defaults,
|
||||||
|
SequenceIdentities: []ck.SequenceIdentity{ck.SequenceIdentityMatching(ck.SequenceSelector{ck.SequenceLiteralKey("glossary_collections"), ck.SequenceAnyMappingKey(), ck.SequenceLiteralKey("entries")}, "id")},
|
||||||
Version: &ck.VersionRule{Path: ck.Keys("version"), Check: func(version ck.Version, v *ck.Validation) error {
|
Version: &ck.VersionRule{Path: ck.Keys("version"), Check: func(version ck.Version, v *ck.Validation) error {
|
||||||
if !version.Present || version.Null || version.Text != "2" {
|
if version.Present && (version.Null || version.Text != "3") {
|
||||||
v.Error(ck.Keys("version"), "version 2 is required; see docs/migrations/v2.md")
|
v.Error(ck.Keys("version"), "use application configuration version 3; see docs/migrations/v3.md")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}},
|
}},
|
||||||
Resolve: func(c *Config, r *ck.Resolution) error {
|
Resolve: resolveSettings,
|
||||||
ck.Default(r, ck.Keys("llm", "validation_profile"), &c.LLM.ValidationProfile, c.LLM.ProposalProfile, "validation uses the final proposal profile when omitted", ck.Keys("llm", "proposal_profile"))
|
|
||||||
for id, backend := range c.Backends {
|
|
||||||
changed := false
|
|
||||||
if !r.IsExplicit(ck.Keys("backends", id, "concurrency")) {
|
|
||||||
backend.Concurrency = 1
|
|
||||||
changed = true
|
|
||||||
}
|
|
||||||
if !r.IsExplicit(ck.Keys("backends", id, "queue_capacity")) {
|
|
||||||
backend.QueueCapacity = 1024
|
|
||||||
changed = true
|
|
||||||
}
|
|
||||||
if changed {
|
|
||||||
c.Backends[id] = backend
|
|
||||||
r.Record(ck.Keys("backends", id), ck.ChangeDerivedDefault, "finite backend defaults")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
Validate: validate,
|
Validate: validate,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -142,21 +155,52 @@ func validate(c *Config, v *ck.Validation) {
|
|||||||
v.Error(path, message)
|
v.Error(path, message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
check(strings.TrimSpace(c.ProfilesDir) != "", ck.Keys("profiles_dir"), "profile directory is required")
|
for name, collection := range c.GlossaryCollections {
|
||||||
check(strings.TrimSpace(c.LLM.ProposalProfile) != "", ck.Keys("llm", "proposal_profile"), "proposal profile is required")
|
for i, entry := range collection.Entries {
|
||||||
check(strings.TrimSpace(c.LLM.ValidationProfile) != "", ck.Keys("llm", "validation_profile"), "validation profile is required")
|
path := ck.Keys("glossary_collections", name, "entries").Index(i)
|
||||||
|
check(entry.ID != "", path.Key("id"), "entry id must not be empty")
|
||||||
|
check(entry.Name != "", path.Key("name"), "entry name must not be empty")
|
||||||
|
check(entry.Category != "", path.Key("category"), "entry category must not be empty")
|
||||||
|
for index, alias := range entry.Aliases {
|
||||||
|
check(alias != "", path.Key("aliases").Index(index), "alias must not be empty")
|
||||||
|
}
|
||||||
|
check(entry.Plural == nil || *entry.Plural != "", path.Key("plural"), "plural must not be empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
|
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
|
||||||
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
|
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
|
||||||
|
check(c.LLM.CorrectionRepairAttempts >= 0, ck.Keys("llm", "correction_repair_attempts"), "correction repair attempts must be non-negative")
|
||||||
|
check(c.LLM.CorrectionRepairProfile == nil || *c.LLM.CorrectionRepairProfile != "", ck.Keys("llm", "correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
|
||||||
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
|
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
|
||||||
check(len(c.Backends) > 0, ck.Keys("backends"), "at least one backend is required")
|
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
|
||||||
for id, b := range c.Backends {
|
validateSelections := func(values []string, path ck.Path) {
|
||||||
p := ck.Keys("backends", id)
|
for i, name := range values {
|
||||||
check(strings.TrimSpace(b.Endpoint) != "", p.Key("endpoint"), "backend endpoint is required")
|
_, exists := c.GlossaryCollections[name]
|
||||||
check(b.Concurrency > 0, p.Key("concurrency"), "backend concurrency must be positive")
|
check(exists, path.Index(i), "unknown glossary collection")
|
||||||
check(b.QueueCapacity >= 0, p.Key("queue_capacity"), "queue capacity must be non-negative")
|
|
||||||
}
|
}
|
||||||
for i, module := range c.Pipeline.Modules {
|
}
|
||||||
check(modulecatalog.IsSupported(module), ck.Keys("pipeline", "modules").Index(i), "unsupported module key")
|
validateSelections(c.Glossaries, ck.Keys("glossaries"))
|
||||||
|
validateSelections(c.ProtectedTerms, ck.Keys("protected_terms"))
|
||||||
|
ids := make(map[string]bool)
|
||||||
|
for i, item := range c.Pipeline.Stages {
|
||||||
|
path := ck.Keys("pipeline", "stages").Index(i)
|
||||||
|
check(modulecatalog.IsSupported(item.Module), path.Key("module"), "unsupported module key")
|
||||||
|
if item.ID != nil {
|
||||||
|
check(*item.ID != "", path.Key("id"), "id must not be empty")
|
||||||
|
check(!ids[*item.ID], path.Key("id"), "id must be unique")
|
||||||
|
ids[*item.ID] = true
|
||||||
|
}
|
||||||
|
validateSelections(item.Glossaries, path.Key("glossaries"))
|
||||||
|
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
|
||||||
|
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
|
||||||
|
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
|
||||||
|
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
|
||||||
|
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
|
||||||
|
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
|
||||||
|
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
|
||||||
|
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")
|
||||||
|
check(item.Validation.MaxPromptTokens > 0, path.Key("validation").Key("max_prompt_tokens"), "validation prompt budget must be positive")
|
||||||
|
check(len(strings.TrimSpace(item.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, path.Key("context").Key("description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
||||||
}
|
}
|
||||||
check(outputschema.IsSupported(c.Output.Schema), ck.Keys("output", "schema"), "unsupported output schema")
|
check(outputschema.IsSupported(c.Output.Schema), ck.Keys("output", "schema"), "unsupported output schema")
|
||||||
check(c.Validation.MaxPromptTokens > 0, ck.Keys("validation", "max_prompt_tokens"), "validation prompt budget must be positive")
|
check(c.Validation.MaxPromptTokens > 0, ck.Keys("validation", "max_prompt_tokens"), "validation prompt budget must be positive")
|
||||||
@@ -172,18 +216,3 @@ func validate(c *Config, v *ck.Validation) {
|
|||||||
}
|
}
|
||||||
check(len(strings.TrimSpace(c.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, ck.Keys("context", "description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
check(len(strings.TrimSpace(c.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, ck.Keys("context", "description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseModulesCSV is used only for an explicitly supplied --modules override.
|
|
||||||
func ParseModulesCSV(raw string) ([]string, error) {
|
|
||||||
if raw == "" {
|
|
||||||
return []string{}, nil
|
|
||||||
}
|
|
||||||
parts := strings.Split(raw, ",")
|
|
||||||
for i := range parts {
|
|
||||||
parts[i] = strings.TrimSpace(parts[i])
|
|
||||||
if parts[i] == "" {
|
|
||||||
return nil, fmt.Errorf("modules list contains an empty value")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parts, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,303 +1,199 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"io/fs"
|
||||||
"errors"
|
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
ck "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const base = "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: proposal\n"
|
func TestDefaultsAndVersion(t *testing.T) {
|
||||||
|
result, err := Load(ck.Input{Source: ck.NoSource()}, "")
|
||||||
func loadText(text, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
|
||||||
return Load(ck.Bytes("config.yml", []byte(text)), overlay, overrides...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultsAndDomainSettings(t *testing.T) {
|
|
||||||
loaded, err := loadText(base, "")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
c := loaded.Config
|
if result.Config.Version != 3 || len(result.Config.Pipeline.Stages) != 5 {
|
||||||
if c.LLM.ValidationProfile != "proposal" || c.Backends["local"].Concurrency != 1 || c.Backends["local"].QueueCapacity != 1024 {
|
t.Fatalf("defaults: %+v", result.Config)
|
||||||
t.Fatalf("derived settings: %+v %+v", c.LLM, c.Backends)
|
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(c.Pipeline.Modules, []string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}) {
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
t.Fatalf("module order: %v", c.Pipeline.Modules)
|
for _, text := range []string{"version: 2\n", "version: null\n", "version: invalid\n"} {
|
||||||
}
|
if err := os.WriteFile(path, []byte(text), 0644); err != nil {
|
||||||
if explanation, ok := loaded.Explain(ck.Keys("llm", "validation_profile")); !ok || len(explanation.History) == 0 {
|
|
||||||
t.Fatal("validation inheritance has no recorded derivation")
|
|
||||||
}
|
|
||||||
if _, err := loadText(base+"pipeline:\n modules: []\n", ""); err != nil {
|
|
||||||
t.Fatalf("empty pipeline: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := loadText(base+"pipeline:\n modules: [grammar, glossary, grammar]\n", ""); err != nil {
|
|
||||||
t.Fatalf("repeated modules: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStrictInputAndConstraints(t *testing.T) {
|
|
||||||
for _, tc := range []struct{ name, text string }{
|
|
||||||
{"old version", strings.Replace(base, "version: 2", "version: 1", 1)},
|
|
||||||
{"missing version", strings.Replace(base, "version: 2\n", "", 1)},
|
|
||||||
{"null version", strings.Replace(base, "version: 2", "version: null", 1)},
|
|
||||||
{"quoted version", strings.Replace(base, "version: 2", "version: '2'", 1)},
|
|
||||||
{"unknown field", base + "mystery: true\n"},
|
|
||||||
{"empty validation profile", base + " validation_profile: ''\n"},
|
|
||||||
{"null validation profile", base + " validation_profile: null\n"},
|
|
||||||
{"null proposal profile", strings.Replace(base, "proposal_profile: proposal", "proposal_profile: null", 1)},
|
|
||||||
{"zero backend concurrency", base + " concurrency: 0\n"},
|
|
||||||
{"invalid target", base + "chunking:\n target_sections: 0\n"},
|
|
||||||
{"invalid section relation", base + "chunking:\n min_section_tokens: 10000\n"},
|
|
||||||
{"string normalization", base + "normalization:\n max_segment_gap: 4s\n"},
|
|
||||||
{"invalid threshold", base + "thresholds:\n glossary: 1.1\n"},
|
|
||||||
{"removed retention", base + "diagnostics:\n retention: always\n"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if _, err := loadText(tc.text, ""); err == nil {
|
|
||||||
t.Fatal("invalid config accepted")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_, err := loadText(strings.Replace(base, "version: 2", "version: 1", 1), "")
|
|
||||||
var loadErr *ck.LoadError
|
|
||||||
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 || !strings.Contains(loadErr.Diagnostics()[0].Message, "docs/migrations/v2.md") {
|
|
||||||
t.Fatalf("migration guidance: %v", err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct{ name, text string }{
|
|
||||||
{"optional target null", base + "chunking:\n target_sections: null\n"},
|
|
||||||
{"zero queue", strings.Replace(base, "llm:\n", " queue_capacity: 0\nllm:\n", 1)},
|
|
||||||
{"zero retries", base + " transient_retries: 0\n repair_attempts: 0\n"},
|
|
||||||
{"zero threshold", base + "thresholds:\n grammar: 0\n"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if _, err := loadText(tc.text, ""); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
})
|
if _, err := Load(ck.Input{Source: ck.File(path)}, ""); err == nil {
|
||||||
|
t.Fatalf("accepted %q", text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := os.WriteFile(path, []byte("pipeline:\n stages: []\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
empty, err := Load(ck.Input{Source: ck.File(path)}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(empty.Config.Pipeline.Stages) != 0 {
|
||||||
|
t.Fatal("empty pipeline was not preserved")
|
||||||
|
}
|
||||||
|
|
||||||
func TestOverlayOverrideAndInheritance(t *testing.T) {
|
}
|
||||||
|
|
||||||
|
func TestMainContextImportsAndOverlay(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
main := filepath.Join(dir, "config.yml")
|
for name, contents := range map[string]string{
|
||||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
"main.yml": "version: 3\nconfig:\n imports:\n - path: fragment.yml\noutput:\n schema: bare-segments\n",
|
||||||
t.Fatal(err)
|
"fragment.yml": "context:\n description: imported\n",
|
||||||
}
|
"overlays/selected.yml": "context:\n description: overlay\n",
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\nchunking:\n max_section_tokens: 1000\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
loaded, err := Load(ck.File(main), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "explicit", Flag: "--proposal-profile"}, ck.Override{Path: ck.Keys("chunking", "min_section_tokens"), Value: 500, Flag: "--min-section-tokens"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if loaded.Config.LLM.ProposalProfile != "explicit" || loaded.Config.LLM.ValidationProfile != "explicit" || loaded.Config.Chunking.MaxSectionTokens != 1000 || loaded.Config.Chunking.MinSectionTokens != 500 {
|
|
||||||
t.Fatalf("resolved: %+v", loaded.Config)
|
|
||||||
}
|
|
||||||
if got, _ := loaded.Metadata().Lookup(ck.Keys("llm", "proposal_profile")); got.Origin.Layer != ck.LayerCLI {
|
|
||||||
t.Fatalf("override origin: %+v", got)
|
|
||||||
}
|
|
||||||
if got, _ := loaded.Metadata().Lookup(ck.Keys("chunking", "max_section_tokens")); got.Origin.Layer != ck.LayerOverlay {
|
|
||||||
t.Fatalf("overlay origin: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSurvivingCredentials(t *testing.T) {
|
|
||||||
t.Setenv("BASE_SECRET", "")
|
|
||||||
t.Setenv("NEW_SECRET", "synthetic-secret")
|
|
||||||
withSecret := strings.Replace(base, "llm:\n", " other:\n endpoint: https://other.invalid/v1\n credential: BASE_SECRET\nllm:\n", 1)
|
|
||||||
if _, err := loadText(withSecret, ""); err == nil {
|
|
||||||
t.Fatal("missing unselected backend secret accepted")
|
|
||||||
}
|
|
||||||
replacement := ck.Override{Path: ck.Keys("backends", "other", "credential"), Value: "NEW_SECRET", Flag: "--credential"}
|
|
||||||
loaded, err := loadText(withSecret, "", replacement)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := loaded.Config.Backends["other"].Credential.Value(); got != "synthetic-secret" {
|
|
||||||
t.Fatalf("resolved secret: %q", got)
|
|
||||||
}
|
|
||||||
var effective bytes.Buffer
|
|
||||||
if err := loaded.WriteEffective(&effective); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(effective.String(), "synthetic-secret") {
|
|
||||||
t.Fatal("effective output leaked a secret")
|
|
||||||
}
|
|
||||||
null := ck.Override{Path: ck.Keys("backends", "other", "credential"), Value: nil, Flag: "--credential"}
|
|
||||||
loaded, err = loadText(withSecret, "", null)
|
|
||||||
if err != nil || loaded.Config.Backends["other"].Credential != nil {
|
|
||||||
t.Fatalf("null credential: %+v %v", loaded, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
inherited := `version: 2
|
|
||||||
backends:
|
|
||||||
parent:
|
|
||||||
endpoint: https://parent.invalid/v1
|
|
||||||
credential: BASE_SECRET
|
|
||||||
local:
|
|
||||||
extends: parent
|
|
||||||
credential: null
|
|
||||||
llm:
|
|
||||||
proposal_profile: proposal
|
|
||||||
`
|
|
||||||
if _, err := loadText(inherited, ""); err == nil {
|
|
||||||
t.Fatal("null child credential removed the unselected parent's surviving reference")
|
|
||||||
}
|
|
||||||
t.Setenv("BASE_SECRET", "synthetic-parent-secret")
|
|
||||||
loaded, err = loadText(strings.Replace(inherited, " credential: null\n", "", 1), "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := loaded.Config.Backends["local"].Credential.Value(); got != "synthetic-parent-secret" {
|
|
||||||
t.Fatalf("inherited secret: %q", got)
|
|
||||||
}
|
|
||||||
if got := loaded.Metadata().IsExplicit(ck.Keys("backends", "local", "credential")); !got {
|
|
||||||
t.Fatal("inherited credential is not explicit")
|
|
||||||
}
|
|
||||||
var inheritedEffective bytes.Buffer
|
|
||||||
if err := loaded.WriteEffective(&inheritedEffective); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(inheritedEffective.String(), "synthetic-parent-secret") {
|
|
||||||
t.Fatal("effective output leaked an inherited secret")
|
|
||||||
}
|
|
||||||
loaded, err = loadText(inherited, "")
|
|
||||||
if err != nil || loaded.Config.Backends["local"].Credential != nil {
|
|
||||||
t.Fatalf("child null credential: %+v %v", loaded, err)
|
|
||||||
}
|
|
||||||
if got := loaded.Config.Backends["parent"].Credential.Value(); got != "synthetic-parent-secret" {
|
|
||||||
t.Fatalf("parent credential after child null: %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("AUDITA_MODEL", "legacy-model")
|
|
||||||
loaded, err = loadText(base, "")
|
|
||||||
if err != nil || loaded.Config.LLM.ProposalProfile != "proposal" {
|
|
||||||
t.Fatalf("legacy environment affected loading: %+v %v", loaded, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSiblingInheritancePreservesBackendSettings(t *testing.T) {
|
|
||||||
text := `version: 2
|
|
||||||
backends:
|
|
||||||
base:
|
|
||||||
endpoint: https://base.invalid/v1
|
|
||||||
concurrency: 3
|
|
||||||
queue_capacity: 0
|
|
||||||
local:
|
|
||||||
extends: base
|
|
||||||
llm:
|
|
||||||
proposal_profile: proposal
|
|
||||||
`
|
|
||||||
loaded, err := loadText(text, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
backend := loaded.Config.Backends["local"]
|
|
||||||
if backend.Endpoint != "https://base.invalid/v1" || backend.Concurrency != 3 || backend.QueueCapacity != 0 {
|
|
||||||
t.Fatalf("inherited backend settings: %+v", backend)
|
|
||||||
}
|
|
||||||
for _, path := range []ck.Path{
|
|
||||||
ck.Keys("backends", "local", "endpoint"),
|
|
||||||
ck.Keys("backends", "local", "concurrency"),
|
|
||||||
ck.Keys("backends", "local", "queue_capacity"),
|
|
||||||
} {
|
} {
|
||||||
if !loaded.Metadata().IsExplicit(path) {
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
t.Errorf("inherited field %s is not explicit", path.String())
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
result, err := Load(ck.Input{Source: ck.File(filepath.Join(dir, "main.yml"))}, "selected", ck.Override{Path: ck.Keys("context", "description"), Value: "override", Flag: "--description"})
|
||||||
overridden, err := loadText(strings.Replace(text, " extends: base\n", " extends: base\n concurrency: 2\n", 1), "")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got := overridden.Config.Backends["local"].Concurrency; got != 2 {
|
if info, ok := result.Metadata().Lookup(ck.Keys("context", "description")); !ok || info.Origin.Layer != ck.LayerCLI {
|
||||||
t.Fatalf("child concurrency override: %d", got)
|
t.Fatalf("override origin = %+v", info)
|
||||||
}
|
}
|
||||||
if got := overridden.Config.Backends["local"].QueueCapacity; got != 0 {
|
if result.Config.Context.Description != "override" {
|
||||||
t.Fatalf("inherited zero queue capacity: %d", got)
|
t.Fatalf("description = %q", result.Config.Context.Description)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, source := range result.Sources() {
|
||||||
|
if source.Layer == ck.LayerMain && source.ImportedFrom == nil {
|
||||||
|
context, ok := result.SourceContext(source.ID)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("main context missing")
|
||||||
|
}
|
||||||
|
data, err := fs.ReadFile(context.FS, path.Join(context.BaseDir, "fragment.yml"))
|
||||||
|
if err != nil || len(data) == 0 {
|
||||||
|
t.Fatalf("main context cannot access companion: %v", err)
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("main source missing")
|
||||||
|
}
|
||||||
|
if _, err := Load(ck.Input{Source: ck.File(filepath.Join(dir, "main.yml"))}, "missing"); err == nil {
|
||||||
|
t.Fatal("missing overlay accepted")
|
||||||
|
}
|
||||||
|
if _, err := Load(ck.Input{Source: ck.NoSource()}, "selected"); err == nil {
|
||||||
|
t.Fatal("overlay without context accepted")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSiblingInheritanceUsesSelectedOverlay(t *testing.T) {
|
func TestDiscoveryAndSelectedFailure(t *testing.T) {
|
||||||
dir := t.TempDir()
|
first, second := t.TempDir(), t.TempDir()
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
search := ck.Searches(ck.SearchDirectory(first), ck.SearchDirectory(second))
|
||||||
t.Fatal(err)
|
load := func() (*ck.Result[Config], error) {
|
||||||
|
return Load(ck.Input{Source: ck.SearchConfig("audita", search)}, "")
|
||||||
}
|
}
|
||||||
main := filepath.Join(dir, "config.yml")
|
result, err := load()
|
||||||
text := `version: 2
|
|
||||||
backends:
|
|
||||||
base:
|
|
||||||
endpoint: https://base.invalid/v1
|
|
||||||
concurrency: 3
|
|
||||||
queue_capacity: 0
|
|
||||||
local:
|
|
||||||
extends: base
|
|
||||||
concurrency: 2
|
|
||||||
llm:
|
|
||||||
proposal_profile: proposal
|
|
||||||
`
|
|
||||||
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("backends:\n base:\n endpoint: https://overlay.invalid/v1\n concurrency: 4\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
loaded, err := Load(ck.File(main), "testing")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
child := loaded.Config.Backends["local"]
|
for _, source := range result.Sources() {
|
||||||
if child.Endpoint != "https://overlay.invalid/v1" || child.Concurrency != 2 || child.QueueCapacity != 0 {
|
if source.Layer == ck.LayerMain {
|
||||||
t.Fatalf("overlay inheritance and child override: %+v", child)
|
t.Fatalf("unexpected main source: %+v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(second, "config.yml"), []byte("context:\n description: selected\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err = load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Config.Context.Description != "selected" {
|
||||||
|
t.Fatalf("description = %q", result.Config.Context.Description)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(first, "config.yml"), []byte("version: 2\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := load(); err == nil {
|
||||||
|
t.Fatal("invalid selected file fell through")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSiblingInheritanceDoesNotBypassBackendValidation(t *testing.T) {
|
func TestOverlayUsesMainDirectoryWithinWiderRoot(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
dir := filepath.Join(root, "project")
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "overlays", "local.yml"), []byte("context:\n description: project\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml")), Filesystem: &ck.FilesystemContext{Root: root}}, "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Config.Context.Description != "project" {
|
||||||
|
t.Fatalf("description = %q", result.Config.Context.Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchemaConstraintsAndExplicitValues(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name, text string
|
||||||
text string
|
valid bool
|
||||||
}{
|
}{
|
||||||
{
|
{"nullable selections", "glossaries: null\nprotected_terms: []\nllm:\n validation_profile: null\n", true},
|
||||||
name: "invalid inherited parent setting",
|
{"repeated modules", "pipeline:\n stages:\n - module: grammar\n - module: grammar\n", true},
|
||||||
text: `version: 2
|
{"removed backends", "backends: {}\n", false},
|
||||||
backends:
|
{"removed debug", "diagnostics:\n debug: true\n", false},
|
||||||
base:
|
{"unknown module", "pipeline:\n stages:\n - module: unknown\n", false},
|
||||||
endpoint: https://base.invalid/v1
|
{"invalid chunk limit", "chunking:\n target_sections: 0\n", false},
|
||||||
concurrency: 0
|
{"invalid threshold", "thresholds:\n grammar: 1.1\n", false},
|
||||||
local:
|
{"null group", "llm: null\n", false},
|
||||||
extends: base
|
{"zero budgets", "llm:\n transient_retries: 0\n repair_attempts: 0\n", true},
|
||||||
llm:
|
|
||||||
proposal_profile: proposal
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid child override",
|
|
||||||
text: `version: 2
|
|
||||||
backends:
|
|
||||||
base:
|
|
||||||
endpoint: https://base.invalid/v1
|
|
||||||
concurrency: 3
|
|
||||||
local:
|
|
||||||
extends: base
|
|
||||||
concurrency: 0
|
|
||||||
llm:
|
|
||||||
proposal_profile: proposal
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
if _, err := loadText(tc.text, ""); err == nil {
|
if err := os.WriteFile(path, []byte(tc.text), 0644); err != nil {
|
||||||
t.Fatal("invalid backend concurrency accepted")
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err := Load(ck.Input{Source: ck.File(path)}, "")
|
||||||
|
if (err == nil) != tc.valid {
|
||||||
|
t.Fatalf("load error: %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOverridesFollowInheritance(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "config.yml")
|
||||||
|
data := `glossary_collections:
|
||||||
|
common:
|
||||||
|
entries:
|
||||||
|
- id: a
|
||||||
|
name: First
|
||||||
|
category: character
|
||||||
|
derived:
|
||||||
|
extends: common
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := Load(ck.Input{Source: ck.File(path)}, "", ck.Override{Path: ck.Keys("glossary_collections", "common", "entries"), Value: []any{map[string]any{"id": "a", "name": "Changed", "category": "character"}}, Flag: "--set"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
|
||||||
|
}
|
||||||
|
if got := result.Config.GlossaryCollections["derived"].Entries[0].Name; got != "First" {
|
||||||
|
t.Fatalf("inherited name = %q", got)
|
||||||
|
}
|
||||||
|
if got := result.Config.GlossaryCollections["common"].Entries[0].Name; got != "Changed" {
|
||||||
|
t.Fatalf("overridden name = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
139
internal/core/config/glossary_test.go
Normal file
139
internal/core/config/glossary_test.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGlossarySourcesComposeAndRetainExtensions(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write := func(name, data string) {
|
||||||
|
t.Helper()
|
||||||
|
file := filepath.Join(dir, name)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(file, []byte(data), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write("config.yml", `version: 3
|
||||||
|
config:
|
||||||
|
imports: []
|
||||||
|
glossary_collections:
|
||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: hero
|
||||||
|
name: Revised
|
||||||
|
category: character
|
||||||
|
level: 5
|
||||||
|
`)
|
||||||
|
write("glossary.yml", `config:
|
||||||
|
imports:
|
||||||
|
- path: shared/extra.yml
|
||||||
|
required: true
|
||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: hero
|
||||||
|
name: Original
|
||||||
|
category: character
|
||||||
|
aliases: [Old]
|
||||||
|
- id: other
|
||||||
|
name: Other
|
||||||
|
category: place
|
||||||
|
`)
|
||||||
|
write("shared/extra.yml", "lore:\n entries:\n - id: hero\n name: Ancient\n category: lore\n")
|
||||||
|
write("glossary/nested/spells.yaml", "spells:\n entries:\n - id: bolt\n name: Bolt\n category: spell\n aliases: [Spark]\n plural: Bolts\n extra: {rank: 2}\n")
|
||||||
|
result, err := Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
|
||||||
|
}
|
||||||
|
party := result.Config.GlossaryCollections["party"].Entries
|
||||||
|
if len(party) != 2 || party[0].Name != "Revised" || len(party[0].Aliases) != 0 || party[1].ID != "other" {
|
||||||
|
t.Fatalf("party=%+v", party)
|
||||||
|
}
|
||||||
|
if party[0].Extra["level"] == nil {
|
||||||
|
t.Fatalf("extension missing: %+v", party[0])
|
||||||
|
}
|
||||||
|
if got := result.Config.GlossaryCollections["lore"].Entries; len(got) != 1 || got[0].ID != "hero" {
|
||||||
|
t.Fatalf("lore=%+v", got)
|
||||||
|
}
|
||||||
|
discovered, err := Load(ck.Input{Source: ck.SearchConfig("audita", ck.SearchDirectory(dir))}, "")
|
||||||
|
if err != nil || len(discovered.Config.GlossaryCollections["party"].Entries) != 2 {
|
||||||
|
t.Fatalf("discovered glossary: %+v %v", discovered, err)
|
||||||
|
}
|
||||||
|
spells := result.Config.GlossaryCollections["spells"].Entries
|
||||||
|
if len(spells) != 1 || spells[0].Plural == nil || *spells[0].Plural != "Bolts" || spells[0].Extra["extra"] == nil {
|
||||||
|
t.Fatalf("spells=%+v", spells)
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for _, source := range result.Sources() {
|
||||||
|
if source.Kind == ck.SourceKindFile {
|
||||||
|
paths = append(paths, source.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range []string{"glossary.yml", "shared/extra.yml", "glossary/nested/spells.yaml"} {
|
||||||
|
if !slices.Contains(paths, filepath.Join(dir, name)) {
|
||||||
|
t.Errorf("missing source %s: %v", name, paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGlossaryValidationAndPeerConflicts(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
write := func(path, data string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(data), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, data := range []string{
|
||||||
|
"glossary_collections:\n empty: {entries: []}\n",
|
||||||
|
"glossary_collections:\n x:\n entries:\n - {id: 'id with spaces', name: N, category: C}\n",
|
||||||
|
} {
|
||||||
|
write(main, data)
|
||||||
|
if _, err := Load(ck.Input{Source: ck.File(main)}, ""); err != nil {
|
||||||
|
t.Fatalf("valid %q: %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, data := range []string{
|
||||||
|
"glossary_collections:\n x:\n entries:\n - {id: '', name: N, category: C}\n",
|
||||||
|
"glossary_collections:\n x:\n entries:\n - {id: x, name: '', category: C}\n",
|
||||||
|
"glossary_collections:\n x:\n entries:\n - {id: x, name: N, category: C, aliases: ['']}\n",
|
||||||
|
"glossary_collections:\n x:\n entries:\n - {id: x, name: N, category: C, plural: ''}\n",
|
||||||
|
} {
|
||||||
|
write(main, data)
|
||||||
|
if _, err := Load(ck.Input{Source: ck.File(main)}, ""); err == nil {
|
||||||
|
t.Fatalf("accepted %q", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write(main, "version: 3\n")
|
||||||
|
write(filepath.Join(dir, "glossary.yml"), "x:\n entries:\n - {id: same, name: One, category: C}\n")
|
||||||
|
if err := os.Mkdir(filepath.Join(dir, "glossary"), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write(filepath.Join(dir, "glossary", "other.yml"), "x:\n entries:\n - {id: same, name: Two, category: C}\n")
|
||||||
|
if _, err := Load(ck.Input{Source: ck.File(main)}, ""); err == nil {
|
||||||
|
t.Fatal("peer conflict accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGlossaryUnderlaySkipsAbsentMain(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "glossary.yml"), []byte("leak:\n entries:\n - {id: a, name: A, category: C}\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Chdir(dir)
|
||||||
|
result, err := Load(ck.Input{Source: ck.NoSource()}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.Config.GlossaryCollections) != 0 {
|
||||||
|
t.Fatalf("unanchored glossary loaded: %+v", result.Config.GlossaryCollections)
|
||||||
|
}
|
||||||
|
}
|
||||||
96
internal/core/config/resolve.go
Normal file
96
internal/core/config/resolve.go
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func cloneSelection(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copied := *value
|
||||||
|
return &copied
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultNames(r *ck.Resolution, path ck.Path, target *[]string, inherited []string, reason string, source ck.Path) {
|
||||||
|
if !r.IsExplicit(path) {
|
||||||
|
ck.Default(r, path, target, slices.Clone(inherited), reason, source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveSettings records root-derived defaults after all explicit assignments.
|
||||||
|
func resolveSettings(c *Config, r *ck.Resolution) error {
|
||||||
|
glossaries := ck.Keys("glossaries")
|
||||||
|
protection := ck.Keys("protected_terms")
|
||||||
|
defaultNames(r, protection, &c.ProtectedTerms, c.Glossaries, "protection defaults to selected glossaries", glossaries)
|
||||||
|
rootValidation := ck.Keys("llm", "validation_profile")
|
||||||
|
rootValidationInfo, _ := r.EntryInfo(rootValidation)
|
||||||
|
inheritedValidation := r.IsExplicit(rootValidation) && !rootValidationInfo.Null && c.LLM.ValidationProfile != nil
|
||||||
|
rootRepair := ck.Keys("llm", "correction_repair_profile")
|
||||||
|
rootRepairInfo, _ := r.EntryInfo(rootRepair)
|
||||||
|
inheritedRepair := r.IsExplicit(rootRepair) && !rootRepairInfo.Null && c.LLM.CorrectionRepairProfile != nil
|
||||||
|
for i := range c.Pipeline.Stages {
|
||||||
|
item := &c.Pipeline.Stages[i]
|
||||||
|
base := ck.Keys("pipeline", "stages").Index(i)
|
||||||
|
defaultNames(r, base.Key("glossaries"), &item.Glossaries, c.Glossaries, "inherit root glossaries", glossaries)
|
||||||
|
defaultNames(r, base.Key("protected_terms"), &item.ProtectedTerms, c.ProtectedTerms, "inherit root protection", protection)
|
||||||
|
threshold := ck.Keys("thresholds", item.Module)
|
||||||
|
var value float64
|
||||||
|
switch item.Module {
|
||||||
|
case "glossary":
|
||||||
|
value = c.Thresholds.Glossary
|
||||||
|
case "homophones":
|
||||||
|
value = c.Thresholds.Homophones
|
||||||
|
case "spoken_word":
|
||||||
|
value = c.Thresholds.SpokenWord
|
||||||
|
case "grammar":
|
||||||
|
value = c.Thresholds.Grammar
|
||||||
|
}
|
||||||
|
ck.Default(r, base.Key("confidence_threshold"), &item.ConfidenceThreshold, value, "inherit module threshold", threshold)
|
||||||
|
llm := base.Key("llm")
|
||||||
|
rootLLM := ck.Keys("llm")
|
||||||
|
proposal := llm.Key("proposal_profile")
|
||||||
|
ck.Default(r, proposal, &item.LLM.ProposalProfile, cloneSelection(c.LLM.ProposalProfile), "inherit proposal profile", rootLLM.Key("proposal_profile"))
|
||||||
|
ck.Default(r, llm.Key("transient_retries"), &item.LLM.TransientRetries, c.LLM.TransientRetries, "inherit retry budget", rootLLM.Key("transient_retries"))
|
||||||
|
ck.Default(r, llm.Key("repair_attempts"), &item.LLM.RepairAttempts, c.LLM.RepairAttempts, "inherit repair budget", rootLLM.Key("repair_attempts"))
|
||||||
|
ck.Default(r, llm.Key("correction_repair_attempts"), &item.LLM.CorrectionRepairAttempts, c.LLM.CorrectionRepairAttempts, "inherit correction repair budget", rootLLM.Key("correction_repair_attempts"))
|
||||||
|
ck.Default(r, llm.Key("operation_timeout"), &item.LLM.OperationTimeout, c.LLM.OperationTimeout, "inherit operation timeout", rootLLM.Key("operation_timeout"))
|
||||||
|
ck.Default(r, base.Key("validation").Key("max_prompt_tokens"), &item.Validation.MaxPromptTokens, c.Validation.MaxPromptTokens, "inherit validation budget", ck.Keys("validation", "max_prompt_tokens"))
|
||||||
|
ck.Default(r, base.Key("context").Key("description"), &item.Context.Description, c.Context.Description, "inherit description", ck.Keys("context", "description"))
|
||||||
|
repair := llm.Key("correction_repair_profile")
|
||||||
|
repairInfo, _ := r.EntryInfo(repair)
|
||||||
|
if r.IsExplicit(repair) && repairInfo.Null {
|
||||||
|
item.LLM.CorrectionRepairProfile = cloneSelection(item.LLM.ProposalProfile)
|
||||||
|
r.RecordDerivedDefault(repair, "correction repair defaults to proposal profile", proposal)
|
||||||
|
} else if inheritedRepair {
|
||||||
|
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.CorrectionRepairProfile), "inherit correction repair profile", rootRepair)
|
||||||
|
} else {
|
||||||
|
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(item.LLM.ProposalProfile), "correction repair defaults to proposal profile", proposal)
|
||||||
|
}
|
||||||
|
validation := llm.Key("validation_profile")
|
||||||
|
info, _ := r.EntryInfo(validation)
|
||||||
|
if r.IsExplicit(validation) && info.Null {
|
||||||
|
item.LLM.ValidationProfile = cloneSelection(item.LLM.ProposalProfile)
|
||||||
|
r.RecordDerivedDefault(validation, "validation defaults to proposal profile", proposal)
|
||||||
|
} else if inheritedValidation {
|
||||||
|
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(c.LLM.ValidationProfile), "inherit validation profile", rootValidation)
|
||||||
|
} else {
|
||||||
|
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(item.LLM.ProposalProfile), "validation defaults to proposal profile", proposal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.IsExplicit(rootRepair) && rootRepairInfo.Null {
|
||||||
|
c.LLM.CorrectionRepairProfile = cloneSelection(c.LLM.ProposalProfile)
|
||||||
|
r.RecordDerivedDefault(rootRepair, "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||||
|
} else {
|
||||||
|
ck.Default(r, rootRepair, &c.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.ProposalProfile), "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||||
|
}
|
||||||
|
if r.IsExplicit(rootValidation) && rootValidationInfo.Null {
|
||||||
|
c.LLM.ValidationProfile = cloneSelection(c.LLM.ProposalProfile)
|
||||||
|
r.RecordDerivedDefault(rootValidation, "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||||
|
} else {
|
||||||
|
ck.Default(r, rootValidation, &c.LLM.ValidationProfile, cloneSelection(c.LLM.ProposalProfile), "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
211
internal/core/config/resolve_test.go
Normal file
211
internal/core/config/resolve_test.go
Normal file
@@ -0,0 +1,211 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadResolution(t *testing.T, text string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
||||||
|
t.Helper()
|
||||||
|
file := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(file, []byte(text), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return Load(ck.Input{Source: ck.File(file)}, "", overrides...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolutionCopiesRootOverridesAndHonorsClears(t *testing.T) {
|
||||||
|
document := `glossary_collections:
|
||||||
|
party: {entries: []}
|
||||||
|
spells: {entries: []}
|
||||||
|
glossaries: [party]
|
||||||
|
llm:
|
||||||
|
proposal_profile: A
|
||||||
|
repair_attempts: 2
|
||||||
|
context:
|
||||||
|
description: root
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
glossaries: [party]
|
||||||
|
llm:
|
||||||
|
proposal_profile: B
|
||||||
|
repair_attempts: 0
|
||||||
|
context:
|
||||||
|
description: ""
|
||||||
|
- module: grammar
|
||||||
|
protected_terms: null
|
||||||
|
- module: grammar
|
||||||
|
glossaries: []
|
||||||
|
`
|
||||||
|
result, err := loadResolution(t, document, ck.Override{Path: ck.Keys("glossaries"), Value: []any{"spells"}, Flag: "--glossaries"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := result.Config
|
||||||
|
if !reflect.DeepEqual(cfg.ProtectedTerms, []string{"spells"}) || !reflect.DeepEqual(cfg.Pipeline.Stages[0].Glossaries, []string{"party"}) || !reflect.DeepEqual(cfg.Pipeline.Stages[1].Glossaries, []string{"spells"}) || len(cfg.Pipeline.Stages[2].Glossaries) != 0 {
|
||||||
|
t.Fatalf("resolved: %+v", cfg)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Stages[1].ProtectedTerms != nil || cfg.Pipeline.Stages[0].LLM.RepairAttempts != 0 || cfg.Pipeline.Stages[0].Context.Description != "" {
|
||||||
|
t.Fatalf("clears lost: %+v", cfg.Pipeline.Stages)
|
||||||
|
}
|
||||||
|
if *cfg.Pipeline.Stages[0].LLM.ValidationProfile != "B" || *cfg.Pipeline.Stages[1].LLM.ValidationProfile != "A" {
|
||||||
|
t.Fatalf("fallback: %+v", cfg.Pipeline.Stages)
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Stages[0].ProtectedTerms[0] = "changed"
|
||||||
|
if cfg.ProtectedTerms[0] != "spells" || cfg.Pipeline.Stages[2].ProtectedTerms[0] != "spells" {
|
||||||
|
t.Fatal("selections share mutable storage")
|
||||||
|
}
|
||||||
|
info, ok := result.Explain(ck.Keys("pipeline", "stages").Index(0).Key("protected_terms"))
|
||||||
|
if !ok || len(info.History) == 0 {
|
||||||
|
t.Fatal("missing resolution evidence")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationProfileFallback(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ name, root, local, want string }{
|
||||||
|
{"proposal override", "", "proposal_profile: B", "B"},
|
||||||
|
{"inherited selection", "validation_profile: V", "proposal_profile: B", "V"},
|
||||||
|
{"explicit null", "validation_profile: V", "proposal_profile: B\n validation_profile: null", "B"},
|
||||||
|
{"root null", "validation_profile: null", "proposal_profile: B", "B"},
|
||||||
|
{"stage selection", "", "validation_profile: W", "W"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
doc := "llm:\n proposal_profile: A\n"
|
||||||
|
if tc.root != "" {
|
||||||
|
doc += " " + tc.root + "\n"
|
||||||
|
}
|
||||||
|
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
|
||||||
|
result, err := loadResolution(t, doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value := result.Config.Pipeline.Stages[0].LLM.ValidationProfile
|
||||||
|
if value == nil || *value != tc.want {
|
||||||
|
t.Fatalf("profile=%v want=%s", value, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvedSettingsValidation(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ name, text string }{
|
||||||
|
{"unknown root collection", "glossaries: [missing]\n"},
|
||||||
|
{"unknown local collection", "pipeline:\n stages:\n - module: grammar\n protected_terms: [missing]\n"},
|
||||||
|
{"empty id", "pipeline:\n stages:\n - {module: grammar, id: ''}\n"},
|
||||||
|
{"duplicate id", "pipeline:\n stages:\n - {module: grammar, id: same}\n - {module: grammar, id: same}\n"},
|
||||||
|
{"empty root validation profile", "llm:\n validation_profile: ''\n"},
|
||||||
|
{"empty local validation profile", "pipeline:\n stages:\n - module: grammar\n llm:\n validation_profile: ''\n"},
|
||||||
|
{"invalid local threshold", "pipeline:\n stages:\n - {module: grammar, confidence_threshold: 1.1}\n"},
|
||||||
|
{"invalid local timeout", "pipeline:\n stages:\n - module: grammar\n llm: {operation_timeout: 0s}\n"},
|
||||||
|
{"invalid local batch budget", "pipeline:\n stages:\n - module: grammar\n validation: {max_prompt_tokens: 0}\n"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := loadResolution(t, tc.text); err == nil {
|
||||||
|
t.Fatal("invalid settings accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := loadResolution(t, "pipeline:\n stages:\n - {module: grammar, id: 'a/b not slug'}\n"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolutionRecordsFieldDependenciesAndExplicitNull(t *testing.T) {
|
||||||
|
doc := `glossary_collections:
|
||||||
|
party: {entries: []}
|
||||||
|
glossaries: [party]
|
||||||
|
llm:
|
||||||
|
proposal_profile: A
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: grammar
|
||||||
|
llm:
|
||||||
|
proposal_profile: B
|
||||||
|
validation_profile: null
|
||||||
|
`
|
||||||
|
result, err := loadResolution(t, doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
base := ck.Keys("pipeline", "stages").Index(0)
|
||||||
|
for _, tc := range []struct{ path, dependency ck.Path }{
|
||||||
|
{ck.Keys("protected_terms"), ck.Keys("glossaries")},
|
||||||
|
{base.Key("glossaries"), ck.Keys("glossaries")},
|
||||||
|
{base.Key("llm").Key("validation_profile"), base.Key("llm").Key("proposal_profile")},
|
||||||
|
} {
|
||||||
|
found := false
|
||||||
|
for _, change := range result.Metadata().History(tc.path) {
|
||||||
|
if change.Origin.Operation != ck.ChangeDerivedDefault {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, dependency := range change.Dependencies {
|
||||||
|
if dependency.String() == tc.dependency.String() {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("missing %s dependency on %s", tc.path.String(), tc.dependency.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info, ok := result.Metadata().Lookup(base.Key("llm").Key("validation_profile"))
|
||||||
|
if !ok || !info.Explicit {
|
||||||
|
t.Fatalf("explicit null lost: %+v", info)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCorrectionRepairSettingsResolution(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, root, local, want string
|
||||||
|
budget int
|
||||||
|
}{
|
||||||
|
{"default", "", "proposal_profile: B", "B", 1},
|
||||||
|
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
|
||||||
|
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
|
||||||
|
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
|
||||||
|
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
doc := "llm:\n proposal_profile: A\n"
|
||||||
|
if tc.root != "" {
|
||||||
|
doc += " " + tc.root + "\n"
|
||||||
|
}
|
||||||
|
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
|
||||||
|
result, err := loadResolution(t, doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := result.Config
|
||||||
|
got := cfg.Pipeline.Stages[0].LLM
|
||||||
|
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
|
||||||
|
t.Fatalf("settings = %+v", got)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := result.WriteEffective(&effective); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
|
||||||
|
t.Fatalf("effective configuration: %s", effective.String())
|
||||||
|
}
|
||||||
|
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
|
||||||
|
t.Fatal("profiles share storage")
|
||||||
|
}
|
||||||
|
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
|
||||||
|
if _, ok := result.Explain(path); !ok {
|
||||||
|
t.Fatal("missing resolution evidence")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
|
||||||
|
if _, err := loadResolution(t, doc); err == nil {
|
||||||
|
t.Errorf("accepted %s", doc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,13 +5,8 @@ const (
|
|||||||
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
|
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
|
||||||
ArtifactNormalizedTranscript = "normalized-transcript.json"
|
ArtifactNormalizedTranscript = "normalized-transcript.json"
|
||||||
ArtifactNormalizationSummary = "normalization-summary.json"
|
ArtifactNormalizationSummary = "normalization-summary.json"
|
||||||
ArtifactChunkingSummary = "chunking-summary.json"
|
|
||||||
ArtifactUtilizationSummary = "utilization-diagnostics.json"
|
|
||||||
ArtifactLLMExecution = "llm-execution.json"
|
ArtifactLLMExecution = "llm-execution.json"
|
||||||
ArtifactSelectedProfiles = "selected-profiles.yaml"
|
|
||||||
ArtifactConfigOrigins = "config-origins.yaml"
|
|
||||||
ArtifactCorrectionLedger = "correction-ledger.json"
|
ArtifactCorrectionLedger = "correction-ledger.json"
|
||||||
ArtifactInvocationMetadata = "invocation.json"
|
ArtifactInvocationMetadata = "invocation.json"
|
||||||
ArtifactEffectiveConfig = "effective-config.redacted.yaml"
|
ArtifactEffectiveConfig = "effective-config.redacted.yaml"
|
||||||
ArtifactErrorLog = "error.log"
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package diagnostics
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -23,6 +24,7 @@ type OutputSelection struct {
|
|||||||
ReportDir string
|
ReportDir string
|
||||||
ReportingEnabled bool
|
ReportingEnabled bool
|
||||||
ProtectedPaths []string
|
ProtectedPaths []string
|
||||||
|
Stdout io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReportBundle struct {
|
type ReportBundle struct {
|
||||||
@@ -47,10 +49,17 @@ type OutputSession struct {
|
|||||||
reports [3]string
|
reports [3]string
|
||||||
enabled bool
|
enabled bool
|
||||||
replace func(string, []byte) error
|
replace func(string, []byte) error
|
||||||
|
temp *os.File
|
||||||
|
tempPath string
|
||||||
|
tempDir string
|
||||||
|
publishPath string
|
||||||
|
direct bool
|
||||||
|
stdout io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
|
// SelectOutput checks active destinations before any output resources are opened.
|
||||||
session := &OutputSession{enabled: selection.ReportingEnabled, replace: atomicReplace}
|
func SelectOutput(selection OutputSelection) (*OutputSession, error) {
|
||||||
|
session := &OutputSession{enabled: selection.ReportingEnabled, replace: atomicReplace, stdout: selection.Stdout}
|
||||||
active := make([]string, 0, 4)
|
active := make([]string, 0, 4)
|
||||||
if selection.TranscriptPath != "" {
|
if selection.TranscriptPath != "" {
|
||||||
session.transcript = selection.TranscriptPath
|
session.transcript = selection.TranscriptPath
|
||||||
@@ -113,14 +122,88 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if session.transcript != "" {
|
return session, nil
|
||||||
if err := preflightTranscript(session.transcript); err != nil {
|
}
|
||||||
|
|
||||||
|
// PrepareOutput validates destinations and acquires the transcript resource.
|
||||||
|
func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
|
||||||
|
session, err := SelectOutput(selection)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := session.Prepare(); err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return session, nil
|
return session, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Prepare opens a sibling temporary file for regular transcript output.
|
||||||
|
func (s *OutputSession) Prepare() error {
|
||||||
|
if s == nil {
|
||||||
|
return fmt.Errorf("output session is required")
|
||||||
|
}
|
||||||
|
if s.transcript == "" || s.temp != nil || s.direct {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := os.Stat(s.transcript)
|
||||||
|
if err == nil && info.IsDir() {
|
||||||
|
return fmt.Errorf("transcript output %q is a directory", s.transcript)
|
||||||
|
}
|
||||||
|
if err == nil && !info.Mode().IsRegular() {
|
||||||
|
s.direct = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("transcript output %q: %w", s.transcript, err)
|
||||||
|
}
|
||||||
|
s.publishPath, err = canonicalDestination(s.transcript)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.tempDir, err = os.MkdirTemp(filepath.Dir(s.publishPath), ".audita-output-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err)
|
||||||
|
}
|
||||||
|
s.tempPath = filepath.Join(s.tempDir, "payload")
|
||||||
|
s.temp, err = os.OpenFile(s.tempPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
|
||||||
|
if err != nil {
|
||||||
|
_ = s.Close()
|
||||||
|
return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err)
|
||||||
|
}
|
||||||
|
if info != nil {
|
||||||
|
if err := s.temp.Chmod(info.Mode().Perm()); err != nil {
|
||||||
|
_ = s.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases unpublished output resources; caller-owned streams remain open.
|
||||||
|
func (s *OutputSession) Close() error {
|
||||||
|
if s == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var errs []error
|
||||||
|
if s.temp != nil {
|
||||||
|
errs = append(errs, s.temp.Close())
|
||||||
|
s.temp = nil
|
||||||
|
}
|
||||||
|
if s.tempPath != "" {
|
||||||
|
if err := os.Remove(s.tempPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
s.tempPath = ""
|
||||||
|
}
|
||||||
|
if s.tempDir != "" {
|
||||||
|
if err := os.Remove(s.tempDir); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
s.tempDir = ""
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *OutputSession) ReportPaths() []string {
|
func (s *OutputSession) ReportPaths() []string {
|
||||||
if s == nil || !s.enabled {
|
if s == nil || !s.enabled {
|
||||||
return nil
|
return nil
|
||||||
@@ -135,24 +218,54 @@ func (s *OutputSession) TranscriptPath() string {
|
|||||||
return s.transcript
|
return s.transcript
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize writes all report destinations before input/model processing. On
|
// Initialize publishes each initial report and attempts failed-status reports if any fail.
|
||||||
// failure it makes one best-effort pass to mark already-created reports failed.
|
func (s *OutputSession) Initialize(initial ReportBundle, failed func([]FileFailure) ReportBundle) error {
|
||||||
func (s *OutputSession) Initialize(initial, failed ReportBundle) error {
|
|
||||||
if s == nil || !s.enabled {
|
if s == nil || !s.enabled {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
created := make([]int, 0, len(s.reports))
|
var failures []FileFailure
|
||||||
for i, payload := range bundleParts(initial) {
|
for i, payload := range bundleParts(initial) {
|
||||||
if err := s.replace(s.reports[i], payload); err != nil {
|
if err := s.replace(s.reports[i], payload); err != nil {
|
||||||
for _, prior := range created {
|
failures = append(failures, FileFailure{Path: s.reports[i], Kind: "report initialization", Err: err})
|
||||||
_ = s.replace(s.reports[prior], bundleParts(failed)[prior])
|
|
||||||
}
|
}
|
||||||
return FileFailure{Path: s.reports[i], Kind: "report initialization", Err: err}
|
|
||||||
}
|
|
||||||
created = append(created, i)
|
|
||||||
}
|
}
|
||||||
|
if len(failures) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if err := s.Close(); err != nil {
|
||||||
|
failures = append(failures, FileFailure{Path: s.transcript, Kind: "output cleanup", Err: err})
|
||||||
|
}
|
||||||
|
if failed != nil {
|
||||||
|
for i, payload := range bundleParts(failed(failures)) {
|
||||||
|
if err := s.replace(s.reports[i], payload); err != nil {
|
||||||
|
failures = append(failures, FileFailure{Path: s.reports[i], Kind: "failed report initialization", Err: err})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
errs := make([]error, 0, len(failures))
|
||||||
|
for _, failure := range failures {
|
||||||
|
errs = append(errs, failure)
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrepareDebug creates optional initial diagnostics and returns any nonfatal problems.
|
||||||
|
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte, secrets ...string) (*RunDirectory, []error) {
|
||||||
|
run, err := NewRunDirectory(workDir, runID, secrets...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, []error{err}
|
||||||
|
}
|
||||||
|
var problems []error
|
||||||
|
if err := run.WriteInvocationMetadata(metadata); err != nil {
|
||||||
|
problems = append(problems, err)
|
||||||
|
}
|
||||||
|
if effective != nil {
|
||||||
|
if err := run.WriteEffectiveConfig(effective); err != nil {
|
||||||
|
problems = append(problems, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return run, problems
|
||||||
|
}
|
||||||
|
|
||||||
// Finalize publishes the transcript first, then errors, recoveries, and Markdown.
|
// Finalize publishes the transcript first, then errors, recoveries, and Markdown.
|
||||||
// An empty transcript payload means processing failed and no transcript is written.
|
// An empty transcript payload means processing failed and no transcript is written.
|
||||||
@@ -162,85 +275,105 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
|
|||||||
return []FileFailure{{Kind: "output finalization", Err: fmt.Errorf("output session is required")}}
|
return []FileFailure{{Kind: "output finalization", Err: fmt.Errorf("output session is required")}}
|
||||||
}
|
}
|
||||||
failures := make([]FileFailure, 0)
|
failures := make([]FileFailure, 0)
|
||||||
if transcript != nil && s.transcript != "" {
|
if transcript != nil {
|
||||||
if err := s.writeTranscript(transcript); err != nil {
|
if err := s.writeTranscript(transcript); err != nil {
|
||||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := s.Close(); err != nil {
|
||||||
|
failures = append(failures, FileFailure{Path: s.transcript, Kind: "output cleanup", Err: err})
|
||||||
|
}
|
||||||
if s.enabled {
|
if s.enabled {
|
||||||
for i, payload := range bundleParts(reports) {
|
for i, payload := range bundleParts(reports) {
|
||||||
if err := s.replace(s.reports[i], payload); err != nil {
|
if err := s.replace(s.reports[i], payload); err != nil {
|
||||||
failures = append(failures, FileFailure{Path: s.reports[i], Kind: "report write", Err: err})
|
failures = append(failures, FileFailure{Path: s.reports[i], Kind: "report write", Err: err})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(failures) > 0 {
|
if len(failures) > 0 && failed != nil {
|
||||||
for i, payload := range bundleParts(failed(failures)) {
|
for i, payload := range bundleParts(failed(failures)) {
|
||||||
_ = s.replace(s.reports[i], payload)
|
if err := s.replace(s.reports[i], payload); err != nil {
|
||||||
|
failures = append(failures, FileFailure{Path: s.reports[i], Kind: "failed report write", Err: err})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return failures
|
return failures
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream destinations use ordinary writes; replacing their directory entry would
|
// Stream destinations are opened only when delivering the transcript.
|
||||||
// destroy the device or pipe. Regular files retain per-file replacement.
|
// Finish retains an earlier processing error while reporting delivery failures.
|
||||||
|
func (s *OutputSession) Finish(primary error, transcript []byte, reports ReportBundle, failed func([]FileFailure) ReportBundle) error {
|
||||||
|
if primary != nil {
|
||||||
|
transcript = nil
|
||||||
|
}
|
||||||
|
failures := s.Finalize(transcript, reports, failed)
|
||||||
|
errs := make([]error, 0, len(failures)+1)
|
||||||
|
errs = append(errs, primary)
|
||||||
|
for _, failure := range failures {
|
||||||
|
errs = append(errs, failure)
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *OutputSession) writeTranscript(payload []byte) error {
|
func (s *OutputSession) writeTranscript(payload []byte) error {
|
||||||
info, err := os.Stat(s.transcript)
|
if s.transcript == "" {
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
if s.stdout == nil {
|
||||||
|
return fmt.Errorf("transcript stdout writer is required")
|
||||||
|
}
|
||||||
|
n, err := s.stdout.Write(payload)
|
||||||
|
if err == nil && n != len(payload) {
|
||||||
|
return io.ErrShortWrite
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err == nil && !info.Mode().IsRegular() {
|
if s.direct {
|
||||||
|
info, err := os.Stat(s.transcript)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.IsDir() || info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("transcript output is no longer a stream destination")
|
||||||
|
}
|
||||||
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
|
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, writeErr := file.Write(payload)
|
n, writeErr := file.Write(payload)
|
||||||
|
if writeErr == nil && n != len(payload) {
|
||||||
|
writeErr = io.ErrShortWrite
|
||||||
|
}
|
||||||
return errors.Join(writeErr, file.Close())
|
return errors.Join(writeErr, file.Close())
|
||||||
}
|
}
|
||||||
return s.replace(s.transcript, payload)
|
if s.temp == nil {
|
||||||
|
return fmt.Errorf("transcript output is not prepared")
|
||||||
|
}
|
||||||
|
n, writeErr := s.temp.Write(payload)
|
||||||
|
if writeErr == nil && n != len(payload) {
|
||||||
|
writeErr = io.ErrShortWrite
|
||||||
|
}
|
||||||
|
closeErr := s.temp.Close()
|
||||||
|
s.temp = nil
|
||||||
|
if err := errors.Join(writeErr, closeErr); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(s.publishPath)
|
||||||
|
if err == nil && !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("transcript output changed type after preparation")
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(s.tempPath, s.publishPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.tempPath = ""
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func bundleParts(bundle ReportBundle) [3][]byte {
|
func bundleParts(bundle ReportBundle) [3][]byte {
|
||||||
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
|
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preflightTranscript(path string) error {
|
|
||||||
info, err := os.Stat(path)
|
|
||||||
if err == nil {
|
|
||||||
if info.IsDir() {
|
|
||||||
return fmt.Errorf("transcript output %q is a directory", path)
|
|
||||||
}
|
|
||||||
if !info.Mode().IsRegular() {
|
|
||||||
// Opening a pipe here can block before its reader is ready.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
|
|
||||||
if openErr != nil {
|
|
||||||
return fmt.Errorf("transcript output %q cannot be opened: %w", path, openErr)
|
|
||||||
}
|
|
||||||
if closeErr := file.Close(); closeErr != nil {
|
|
||||||
return fmt.Errorf("transcript output %q cannot be closed: %w", path, closeErr)
|
|
||||||
}
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fmt.Errorf("transcript output %q: %w", path, err)
|
|
||||||
}
|
|
||||||
parent := filepath.Dir(path)
|
|
||||||
probe, err := os.CreateTemp(parent, ".audita-probe-*")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("transcript output parent %q: %w", parent, err)
|
|
||||||
}
|
|
||||||
name := probe.Name()
|
|
||||||
closeErr := probe.Close()
|
|
||||||
removeErr := os.Remove(name)
|
|
||||||
if closeErr != nil {
|
|
||||||
return fmt.Errorf("transcript output probe close: %w", closeErr)
|
|
||||||
}
|
|
||||||
if removeErr != nil {
|
|
||||||
return fmt.Errorf("transcript output probe removal: %w", removeErr)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sameDestination(a, b string) (bool, error) {
|
func sameDestination(a, b string) (bool, error) {
|
||||||
ca, err := canonicalDestination(a)
|
ca, err := canonicalDestination(a)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
package diagnostics
|
package diagnostics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -118,7 +121,7 @@ func TestDisabledReportingIgnoresUnusableDirectory(t *testing.T) {
|
|||||||
if err != nil || len(session.ReportPaths()) != 0 {
|
if err != nil || len(session.ReportPaths()) != 0 {
|
||||||
t.Fatalf("disabled reports probed: %+v %v", session, err)
|
t.Fatalf("disabled reports probed: %+v %v", session, err)
|
||||||
}
|
}
|
||||||
if err := session.Initialize(ReportBundle{}, ReportBundle{}); err != nil {
|
if err := session.Initialize(ReportBundle{}, nil); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(dir, "missing")); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Stat(filepath.Join(dir, "missing")); !errors.Is(err, os.ErrNotExist) {
|
||||||
@@ -144,7 +147,7 @@ func TestStdoutModeInitializesAndFinalizesReports(t *testing.T) {
|
|||||||
}
|
}
|
||||||
initial := ReportBundle{Errors: []byte("in progress errors"), Recoveries: []byte("in progress recoveries"), Markdown: []byte("in progress report")}
|
initial := ReportBundle{Errors: []byte("in progress errors"), Recoveries: []byte("in progress recoveries"), Markdown: []byte("in progress report")}
|
||||||
finished := ReportBundle{Errors: []byte("completed errors"), Recoveries: []byte("completed recoveries"), Markdown: []byte("completed report")}
|
finished := ReportBundle{Errors: []byte("completed errors"), Recoveries: []byte("completed recoveries"), Markdown: []byte("completed report")}
|
||||||
if err := session.Initialize(initial, ReportBundle{}); err != nil {
|
if err := session.Initialize(initial, nil); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if session.TranscriptPath() != "" {
|
if session.TranscriptPath() != "" {
|
||||||
@@ -181,7 +184,12 @@ func TestInitializeFailureMarksPriorReportFailedBeforeModelWork(t *testing.T) {
|
|||||||
return actual(path, payload)
|
return actual(path, payload)
|
||||||
}
|
}
|
||||||
modelCalled := false
|
modelCalled := false
|
||||||
if err := session.Initialize(ReportBundle{Errors: []byte("initial"), Recoveries: []byte("initial"), Markdown: []byte("initial")}, ReportBundle{Errors: []byte("failed"), Recoveries: []byte("failed"), Markdown: []byte("failed")}); err == nil {
|
if err := session.Initialize(ReportBundle{Errors: []byte("initial"), Recoveries: []byte("initial"), Markdown: []byte("initial")}, func(failures []FileFailure) ReportBundle {
|
||||||
|
if len(failures) != 1 || filepath.Base(failures[0].Path) != RecoveriesFile {
|
||||||
|
t.Fatalf("initialization failures: %+v", failures)
|
||||||
|
}
|
||||||
|
return ReportBundle{Errors: []byte("failed"), Recoveries: []byte("failed"), Markdown: []byte("failed")}
|
||||||
|
}); err == nil {
|
||||||
modelCalled = true
|
modelCalled = true
|
||||||
}
|
}
|
||||||
if modelCalled {
|
if modelCalled {
|
||||||
@@ -191,8 +199,8 @@ func TestInitializeFailureMarksPriorReportFailedBeforeModelWork(t *testing.T) {
|
|||||||
if err != nil || string(got) != "failed" {
|
if err != nil || string(got) != "failed" {
|
||||||
t.Fatalf("prior report not failed: %s %v", got, err)
|
t.Fatalf("prior report not failed: %s %v", got, err)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(dir, MarkdownFile)); !errors.Is(err, os.ErrNotExist) {
|
if got, err := os.ReadFile(filepath.Join(dir, MarkdownFile)); err != nil || string(got) != "failed" {
|
||||||
t.Fatalf("later report created: %v", err)
|
t.Fatalf("later report not finalized: %q %v", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,7 +211,7 @@ func TestFinalizationCollectsFailuresAndRetainsPublishedTranscript(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := session.Initialize(ReportBundle{Errors: []byte("initial"), Recoveries: []byte("initial"), Markdown: []byte("initial")}, ReportBundle{}); err != nil {
|
if err := session.Initialize(ReportBundle{Errors: []byte("initial"), Recoveries: []byte("initial"), Markdown: []byte("initial")}, nil); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
actual := session.replace
|
actual := session.replace
|
||||||
@@ -255,3 +263,228 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
|
|||||||
t.Fatalf("debug directory removed: %v", err)
|
t.Fatalf("debug directory removed: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSelectionAndAbortPreserveTranscript(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
output := filepath.Join(dir, "output.json")
|
||||||
|
if err := os.WriteFile(output, []byte("original"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := SelectOutput(OutputSelection{TranscriptPath: output})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(dir); err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("selection changed directory: %v %v", entries, err)
|
||||||
|
}
|
||||||
|
if err := session.Prepare(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if session.temp == nil {
|
||||||
|
t.Fatal("temporary output not open")
|
||||||
|
}
|
||||||
|
temp := session.tempPath
|
||||||
|
if err := session.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(temp); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("unpublished file remains: %v", err)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(output); err != nil || string(data) != "original" {
|
||||||
|
t.Fatalf("output changed: %q %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStdoutAndDeviceDelivery(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
session, err := PrepareOutput(OutputSelection{Stdout: &stdout})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil); len(failures) != 0 {
|
||||||
|
t.Fatal(failures)
|
||||||
|
}
|
||||||
|
if stdout.String() != "transcript" {
|
||||||
|
t.Fatalf("stdout=%q", stdout.String())
|
||||||
|
}
|
||||||
|
session, err = PrepareOutput(OutputSelection{TranscriptPath: os.DevNull})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !session.direct || session.temp != nil {
|
||||||
|
t.Fatal("device was not deferred")
|
||||||
|
}
|
||||||
|
if failures := session.Finalize([]byte("discard"), ReportBundle{}, nil); len(failures) != 0 {
|
||||||
|
t.Fatal(failures)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFinishRetainsPrimaryFailureAndAttemptsReports(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
output := filepath.Join(dir, "transcript.json")
|
||||||
|
if err := os.WriteFile(output, []byte("original"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: output, ReportDir: dir, ReportingEnabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
originalReplace := session.replace
|
||||||
|
session.replace = func(path string, data []byte) error {
|
||||||
|
if filepath.Base(path) == ErrorsFile {
|
||||||
|
return os.ErrPermission
|
||||||
|
}
|
||||||
|
return originalReplace(path, data)
|
||||||
|
}
|
||||||
|
primary := errors.New("processing failed")
|
||||||
|
bundle := ReportBundle{Errors: []byte("failed"), Recoveries: []byte("failed"), Markdown: []byte("failed")}
|
||||||
|
result := session.Finish(primary, []byte("unwanted"), bundle, func([]FileFailure) ReportBundle { return bundle })
|
||||||
|
if !errors.Is(result, primary) || !errors.Is(result, os.ErrPermission) {
|
||||||
|
t.Fatalf("combined error: %v", result)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(output); err != nil || string(data) != "original" {
|
||||||
|
t.Fatalf("output=%q err=%v", data, err)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(filepath.Join(dir, MarkdownFile)); err != nil || string(data) != "failed" {
|
||||||
|
t.Fatalf("remaining report=%q err=%v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
session, err := SelectOutput(OutputSelection{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
file := filepath.Join(dir, "not-directory")
|
||||||
|
if err := os.WriteFile(file, []byte("x"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 {
|
||||||
|
t.Fatalf("run=%v problems=%v", run, problems)
|
||||||
|
}
|
||||||
|
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("secret: sensitive\n"), "sensitive")
|
||||||
|
if len(problems) != 0 {
|
||||||
|
t.Fatal(problems)
|
||||||
|
}
|
||||||
|
contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactEffectiveConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(contents), "sensitive") {
|
||||||
|
t.Fatal("debug artifact contains secret")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type shortOutput struct{}
|
||||||
|
|
||||||
|
func (shortOutput) Write(b []byte) (int, error) { return len(b) / 2, nil }
|
||||||
|
|
||||||
|
func TestShortStdoutWriteIsFailure(t *testing.T) {
|
||||||
|
session, err := PrepareOutput(OutputSelection{Stdout: shortOutput{}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil)
|
||||||
|
if len(failures) != 1 || !errors.Is(failures[0], io.ErrShortWrite) {
|
||||||
|
t.Fatalf("failures=%v", failures)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscriptWriteFailureCleansTemporaryResource(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
output := filepath.Join(dir, "out.json")
|
||||||
|
if err := os.WriteFile(output, []byte("original"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: output})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
temp := session.tempPath
|
||||||
|
if err := session.temp.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
failures := session.Finalize([]byte("new"), ReportBundle{}, nil)
|
||||||
|
if len(failures) == 0 {
|
||||||
|
t.Fatal("closed output accepted")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(temp); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("temporary output remains: %v", err)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(output); err != nil || string(data) != "original" {
|
||||||
|
t.Fatalf("output=%q err=%v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNamedPipeIsNotOpenedDuringPreparation(t *testing.T) {
|
||||||
|
pipe := filepath.Join(t.TempDir(), "output-pipe")
|
||||||
|
makeNamedPipe(t, pipe)
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: pipe})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !session.direct || session.temp != nil {
|
||||||
|
t.Fatal("pipe was not deferred")
|
||||||
|
}
|
||||||
|
if err := session.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLateNamedPipeIsPreserved(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "output")
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
makeNamedPipe(t, path)
|
||||||
|
err = session.Finish(nil, []byte("transcript"), ReportBundle{}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "changed type") {
|
||||||
|
t.Fatalf("expected clear delivery failure, got %v", err)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil || info.Mode()&os.ModeNamedPipe == 0 {
|
||||||
|
t.Fatalf("late pipe was lost: %v %v", info, err)
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("unpublished output resources remain: %v %v", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStreamChangedToRegularFileIsPreserved(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "output")
|
||||||
|
makeNamedPipe(t, path)
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte("existing content"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = session.Finish(nil, []byte("new"), ReportBundle{}, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no longer a stream") {
|
||||||
|
t.Fatalf("expected clear delivery failure, got %v", err)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(path); err != nil || string(data) != "existing content" {
|
||||||
|
t.Fatalf("replacement file changed: %q %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeNamedPipe(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
tool, err := exec.LookPath("mkfifo")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("mkfifo is unavailable")
|
||||||
|
}
|
||||||
|
if output, err := exec.Command(tool, path).CombinedOutput(); err != nil {
|
||||||
|
t.Fatalf("mkfifo: %v %s", err, output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
@@ -25,14 +24,9 @@ type RunDirectory struct {
|
|||||||
type InvocationMetadata struct {
|
type InvocationMetadata struct {
|
||||||
Operation string `json:"operation"`
|
Operation string `json:"operation"`
|
||||||
TranscriptPath string `json:"transcript_path"`
|
TranscriptPath string `json:"transcript_path"`
|
||||||
GlossaryPath string `json:"glossary_path"`
|
|
||||||
OutputPath string `json:"output_path,omitempty"`
|
OutputPath string `json:"output_path,omitempty"`
|
||||||
ConfigPath string `json:"config_path,omitempty"`
|
|
||||||
ConfigSource string `json:"config_source,omitempty"`
|
|
||||||
Overlay string `json:"overlay,omitempty"`
|
Overlay string `json:"overlay,omitempty"`
|
||||||
ConfigVersion *int `json:"config_version,omitempty"`
|
ConfigVersion *int `json:"config_version,omitempty"`
|
||||||
TranscriptDescription string `json:"transcript_description,omitempty"`
|
|
||||||
Modules []string `json:"modules"`
|
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
StartedAt time.Time `json:"started_at"`
|
StartedAt time.Time `json:"started_at"`
|
||||||
}
|
}
|
||||||
@@ -158,26 +152,6 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteErrorLog writes an error log on failure
|
|
||||||
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
|
||||||
errorPath := filepath.Join(r.path, ArtifactErrorLog)
|
|
||||||
return os.WriteFile(errorPath, []byte(r.redactor.String(errorMessage)+"\n"), 0o644)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteChunkingSummary writes the chunking summary artifact
|
|
||||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
|
||||||
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
|
|
||||||
bytes, err := r.redactor.JSON(summary)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
|
||||||
}
|
|
||||||
bytes = append(bytes, '\n')
|
|
||||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
|
||||||
return fmt.Errorf("failed to write chunking summary: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
||||||
artifactPath := filepath.Join(r.path, name)
|
artifactPath := filepath.Join(r.path, name)
|
||||||
bytes, err := r.redactor.JSON(payload)
|
bytes, err := r.redactor.JSON(payload)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user