diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5359c02..771bdab 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -140,6 +140,12 @@ Exercise final v3 examples after the glossary fields are connected. ## Stage 3: Build the shared PromptKit engine and account for its sources +**Completed.** The engine uses recorded application context and captures +operator source and message paths. Request adapters, role selection, CLI +acquisition, and their older tests still require the following assignments. +The focused engine checks were run independently while those callers remain +unconverted; assembled integration is deferred to the later handoff. + Use the [PromptKit outline](promptkit.md), especially its [automatic source contract](promptkit.md#automatic-prompt-sources--resolved). Own engine assembly in `internal/framework/llm/runtime.go`, `source_paths.go`, and embedded assets/registry under `internal/prompts`. diff --git a/internal/framework/llm/engine_sources_test.go b/internal/framework/llm/engine_sources_test.go new file mode 100644 index 0000000..49b1975 --- /dev/null +++ b/internal/framework/llm/engine_sources_test.go @@ -0,0 +1,163 @@ +package llm + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" + ck "go.maximumdirect.net/configkit" +) + +func TestEngineCapturesAutomaticPromptSources(t *testing.T) { + dir := t.TempDir() + main := filepath.Join(dir, "config.yml") + write := func(name, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(text), 0644); err != nil { + t.Fatal(err) + } + } + write("config.yml", "version: 3\n") + write("prompts.yml", "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content_file: message.txt\n output:\n format: text\n validation_mode: none\n") + write("message.txt", "first") + write("prompts/nested/extra.yml", "prompts:\n - id: extra\n messages:\n - role: user\n content: extra\n output:\n format: text\n validation_mode: none\n") + loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") + if err != nil { + t.Fatal(err) + } + first, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) + } + inspect := func(r *Runtime) string { + t.Helper() + p, err := r.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGrammarProposal) + if err != nil { + t.Fatal(err) + } + return p.Messages[0].Content + } + if got := inspect(first); got != "first" { + t.Fatalf("prompt = %q", got) + } + for _, name := range []string{"prompts.yml", "message.txt", "prompts/nested/extra.yml"} { + if !slices.Contains(first.SourcePaths, filepath.Join(dir, name)) { + t.Errorf("missing %s in %v", name, first.SourcePaths) + } + } + if _, err := first.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGlossaryProposal); err != nil { + t.Fatal(err) + } + if _, err := first.Engine.InspectPrompt(context.Background(), "extra"); err != nil { + t.Fatal(err) + } + write("message.txt", "second") + second, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) + } + if inspect(first) != "first" || inspect(second) != "second" { + t.Fatal("engine did not capture source at construction") + } +} + +func TestEngineWithoutMainUsesEmbeddedPrompts(t *testing.T) { + loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "") + if err != nil { + t.Fatal(err) + } + runtime, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) + } + if len(runtime.SourcePaths) != 0 { + t.Fatalf("unexpected paths: %v", runtime.SourcePaths) + } + if _, err := runtime.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGrammarProposal); err != nil { + t.Fatal(err) + } +} + +func TestEngineCapturesInheritedMessageAndRejectsPeerConflicts(t *testing.T) { + dir := t.TempDir() + write := func(name, text string) { + t.Helper() + file := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(text), 0644); err != nil { + t.Fatal(err) + } + } + write("config.yml", "version: 3\n") + write("prompts.yml", "config:\n imports:\n - path: shared/catalog.yml\n required: true\n") + write("shared/catalog.yml", "prompts:\n - id: parent\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n") + write("shared/instruction.md", "inherited content") + write("prompts/child.yml", "prompts:\n - id: child\n extends: {id: parent}\n") + loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") + if err != nil { + t.Fatal(err) + } + runtime, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) + } + prompt, err := runtime.Engine.InspectPrompt(context.Background(), "child") + if err != nil || prompt.Messages[0].Content != "inherited content" { + t.Fatalf("inherited prompt: %+v, %v", prompt, err) + } + for _, name := range []string{"prompts.yml", "shared/catalog.yml", "shared/instruction.md", "prompts/child.yml"} { + if !slices.Contains(runtime.SourcePaths, filepath.Join(dir, name)) { + t.Errorf("missing %s in %v", name, runtime.SourcePaths) + } + } + write("prompts/duplicate.yml", "prompts:\n - id: parent\n messages:\n - role: user\n content: duplicate\n output:\n format: text\n validation_mode: none\n") + if _, err := NewRuntime(context.Background(), loaded, nil); err == nil || IsAssetError(err) { + t.Fatalf("peer conflict classification: %v", err) + } +} + +func TestEngineUsesRecordedContextWithWiderRoot(t *testing.T) { + root := t.TempDir() + project := filepath.Join(root, "campaign") + if err := os.Mkdir(project, 0755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{ + "config.yml": "version: 3\n", + "prompts.yml": "config:\n imports:\n - path: ../shared.yml\n required: true\n", + } { + if err := os.WriteFile(filepath.Join(project, name), []byte(body), 0644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(root, "shared.yml"), []byte("prompts:\n - id: shared\n messages:\n - role: user\n content: shared\n output:\n format: text\n validation_mode: none\n"), 0644); err != nil { + t.Fatal(err) + } + source := ck.SearchConfig("audita", ck.SearchDirectory(project)) + loaded, err := config.Load(ck.Input{Source: source, Filesystem: &ck.FilesystemContext{Root: root}}, "") + if err != nil { + t.Fatal(err) + } + t.Chdir(t.TempDir()) + runtime, err := NewRuntime(context.Background(), loaded, nil) + if err != nil { + t.Fatal(err) + } + if _, err := runtime.Engine.InspectPrompt(context.Background(), "shared"); err != nil { + t.Fatal(err) + } + for _, file := range []string{filepath.Join(project, "prompts.yml"), filepath.Join(root, "shared.yml")} { + if !slices.Contains(runtime.SourcePaths, file) { + t.Errorf("missing %s in %v", file, runtime.SourcePaths) + } + } +} diff --git a/internal/framework/llm/runtime.go b/internal/framework/llm/runtime.go index fc2e5f8..42d7a94 100644 --- a/internal/framework/llm/runtime.go +++ b/internal/framework/llm/runtime.go @@ -10,147 +10,108 @@ import ( "strings" "sync/atomic" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" ) -// Runtime owns one PromptKit engine and the validated selections for both roles. -// The same engine is shared by the role-bound clients during processing. +// Runtime owns the captured engine shared by all processing clients. type Runtime struct { - Config config.Config - Engine *pk.Engine - Profiles map[string]pk.ProfileInspection - // SourcePaths contains host configuration and message files captured by PromptKit. + Config config.Config + Engine *pk.Engine SourcePaths []string nextCallID atomic.Uint64 } -// AssetError indicates a broken embedded prompt or schema rather than an -// operator-provided setting or profile. type AssetError struct{ Err error } func (e *AssetError) Error() string { return e.Err.Error() } func (e *AssetError) Unwrap() error { return e.Err } -func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) { - return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem()) +const operatorFilesystem = "Audita operator prompts" +const operatorImports = `config: + imports: + - path: prompts.yml + required: false + - dir: prompts + required: false + recursive: true + pattern: ["*.yml", "*.yaml"] +` + +func NewRuntime(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*Runtime, error) { + return newRuntimeWithPromptFS(ctx, loaded, client, prompts.Filesystem()) } -func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) { +func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*Runtime, error) { if err := ctx.Err(); err != nil { return nil, err } - profileDir := c.ProfilesDir - if !filepath.IsAbs(profileDir) { - profileDir = filepath.Join(filepath.Dir(mainPath), profileDir) + if loaded == nil { + return nil, fmt.Errorf("application configuration is required") } - promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot) + embedded, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot) if err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr - } return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)} } - profileInputs, err := pk.DirectoryInputs(profileDir) - if err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr + sources := pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: embedded}}} + var hostRoot string + for _, source := range loaded.Sources() { + if source.Layer != ck.LayerMain || source.ImportedFrom != nil { + continue } - return nil, fmt.Errorf("discover operator definitions: %w", err) - } - if err := ctx.Err(); err != nil { - return nil, err - } - applicationBackends := make(map[string]any, len(c.Backends)) - for id := range c.Backends { - backend := c.Backends[id] - applicationBackends[id] = map[string]any{ - "endpoint": backend.Endpoint, - "concurrency_limit": backend.Concurrency, - "queue_capacity": backend.QueueCapacity, - "api_key_env": nil, + context, ok := loaded.SourceContext(source.ID) + if !ok { + break } + filesystem := ck.NewFilesystem(operatorFilesystem, context.FS) + sources.Underlays = append(sources.Underlays, ck.Underlay{Inputs: []ck.Input{{ + Source: ck.Bytes("Audita automatic prompt imports", []byte(operatorImports)), Imports: true, + Filesystem: &ck.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: context.BaseDir}, + }}}) + if source.Kind == ck.SourceKindFile && source.Path != "" { + hostRoot = filepath.Dir(source.Path) + if context.BaseDir != "." { + for range strings.Split(context.BaseDir, "/") { + hostRoot = filepath.Dir(hostRoot) + } + } + } + break } - sources := pk.ConfigurationSources{ - Underlays: []ck.Underlay{ - {Inputs: promptInputs}, - {Inputs: profileInputs}, - }, - Main: &ck.Input{Source: ck.Mapping("application backends", map[string]any{"backends": applicationBackends})}, - } - engine, err := pk.NewEngine(pk.Config{ - Sources: sources, - HTTPClient: client, - TransientRetries: &c.LLM.TransientRetries, - }, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) + engine, err := pk.NewEngine(pk.Config{Sources: sources, HTTPClient: client}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) if err != nil { - return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err)) + return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("assemble LLM runtime: %w", err)) } for _, meta := range prompts.RegisteredMetadata() { - if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil { - cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err) - return nil, embeddedAssetOrInputError(ctx, promptInputs, cause) + if _, err := engine.InspectPrompt(ctx, meta.PromptID); err != nil { + return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)) } } - runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)} - for _, selection := range []struct{ role, id string }{{"proposal", c.LLM.ProposalProfile}, {"validation", c.LLM.ValidationProfile}} { - profile, err := engine.InspectProfile(ctx, selection.id) - if err != nil { - return nil, fmt.Errorf("inspect %s profile: %w", selection.role, err) - } - target := profile.EffectiveModelParams - backend, ok := c.Backends[target.BackendID] - if !ok { - return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role) - } - if strings.TrimSpace(target.Model) == "" { - return nil, fmt.Errorf("%s profile must select a nonblank model", selection.role) - } - if strings.TrimSpace(target.Endpoint) == "" { - return nil, fmt.Errorf("%s profile must resolve to a nonblank endpoint", selection.role) - } - endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") } - if endpoint(target.Endpoint) != endpoint(backend.Endpoint) { - return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role) - } - if target.APIKeyEnv != "" { - return nil, fmt.Errorf("%s profile must use the backend credential, not its own environment lookup", selection.role) - } - if profile.APIKeyRequired && backend.Credential == nil { - return nil, fmt.Errorf("%s profile requires a backend credential", selection.role) - } - runtime.Profiles[selection.role] = *profile - } - runtime.SourcePaths, err = capturedSourcePaths(engine) + paths, err := capturedSourcePaths(engine, hostRoot) if err != nil { return nil, fmt.Errorf("inspect captured LLM sources: %w", err) } - return runtime, nil + return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil } -func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error { +func embeddedAssetOrInputError(ctx context.Context, inputs []ck.Input, cause error) error { if err := ctx.Err(); err != nil { return err } - embeddedSources := pk.ConfigurationSources{ - Underlays: []ck.Underlay{{Inputs: promptInputs}}, - } - diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources}, - pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) + engine, err := pk.NewEngine(pk.Config{Sources: pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: inputs}}}}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) if err == nil { for _, meta := range prompts.RegisteredMetadata() { - if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil { - err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr) + if _, err = engine.InspectPrompt(ctx, meta.PromptID); err != nil { break } } } - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr + if err := ctx.Err(); err != nil { + return err } if err != nil { return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)} @@ -160,8 +121,6 @@ func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cau func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) } -// RedactError replaces every configured resolved backend secret before a -// library error is surfaced in CLI output or an artifact. func RedactError(err error, c config.Config) string { if err == nil { return "" diff --git a/internal/framework/llm/source_paths.go b/internal/framework/llm/source_paths.go index 9de79d7..9c9e350 100644 --- a/internal/framework/llm/source_paths.go +++ b/internal/framework/llm/source_paths.go @@ -7,12 +7,12 @@ import ( "strconv" "strings" - ck "gitea.maximumdirect.net/eric/ConfigKit" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" "gopkg.in/yaml.v3" ) -func capturedSourcePaths(engine *pk.Engine) ([]string, error) { +func capturedSourcePaths(engine *pk.Engine, hostRoot string) ([]string, error) { inspection, err := engine.InspectConfiguration() if err != nil { return nil, fmt.Errorf("inspect configuration: %w", err) @@ -21,20 +21,36 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) { sources := inspection.Sources() byID := make(map[string]ck.SourceInfo, len(sources)) paths := make(map[string]struct{}, len(sources)) + hostPath := func(source ck.SourceInfo) string { + if source.Kind == ck.SourceKindFile { + return source.Path + } + if hostRoot != "" && source.Kind == ck.SourceKindFilesystem && strings.HasPrefix(source.Label, operatorFilesystem+"/") { + return filepath.Join(hostRoot, filepath.FromSlash(source.Path)) + } + return "" + } for _, source := range sources { byID[source.ID] = source - if source.Kind == ck.SourceKindFile && source.Path != "" { - paths[source.Path] = struct{}{} + if path := hostPath(source); path != "" { + paths[path] = struct{}{} } } metadata := inspection.Metadata() for _, path := range metadata.HistoricalPaths() { segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/") - if len(segments) != 6 || segments[0] != "prompts" || segments[3] != "messages" || segments[5] != "content_file" { + // Prompt resources are lists at the root or inside a collection. + if len(segments) == 7 && segments[0] == "collections" { + segments = segments[2:] + } + if len(segments) != 5 || segments[0] != "prompts" || segments[2] != "messages" || segments[4] != "content_file" { continue } - if messageIndex, err := strconv.Atoi(segments[4]); err != nil || messageIndex < 0 { + if _, err := strconv.Atoi(segments[1]); err != nil { + continue + } + if _, err := strconv.Atoi(segments[3]); err != nil { continue } field, ok := metadata.Lookup(path) @@ -42,7 +58,7 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) { continue } source, ok := byID[field.Origin.Locations[0].SourceID] - if !ok || source.Kind != ck.SourceKindFile || source.Path == "" { + if !ok || hostPath(source) == "" { continue } explanation, ok := inspection.Explain(path) @@ -53,7 +69,7 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) { if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil { return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err) } - paths[filepath.Join(filepath.Dir(source.Path), filepath.FromSlash(resource))] = struct{}{} + paths[filepath.Join(filepath.Dir(hostPath(source)), filepath.FromSlash(resource))] = struct{}{} } result := make([]string, 0, len(paths)) diff --git a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml index 6b28ebd..49fe96f 100644 --- a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml @@ -1,38 +1,39 @@ -prompts: - modules.glossary.proposal: - v1: - messages: - - role: system - content: | - You are Audita, a careful transcript correction assistant. Identify only transcription errors that are strongly supported by the glossary. A valid correction must be acoustically plausible: the original transcript text should sound similar to the proposed correction when spoken aloud. Do not make generic grammar, spelling, capitalization, style, or filler-word edits. Do not substitute an unrelated glossary term just because it could fit the topic. Preserve speaker names, timestamps, and meaning. - - role: user - content: | - Review this transcript section and return only glossary-supported corrections that should be applied. +collections: + audita: + prompts: + - id: modules.glossary.proposal + messages: + - role: system + content: | + You are Audita, a careful transcript correction assistant. Identify only transcription errors that are strongly supported by the glossary. A valid correction must be acoustically plausible: the original transcript text should sound similar to the proposed correction when spoken aloud. Do not make generic grammar, spelling, capitalization, style, or filler-word edits. Do not substitute an unrelated glossary term just because it could fit the topic. Preserve speaker names, timestamps, and meaning. + - role: user + content: | + Review this transcript section and return only glossary-supported corrections that should be applied. - Rules: - - Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction. - - The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English. - - Appropriate example: correcting "gestures" to "Jesters" can be valid if "Jesters" appears in the glossary and nearby context supports that inference. - - Inappropriate example: correcting "Lyra" to "Jesters" should be omitted because those words are not similar in spoken English, even if "Jesters" appears in the glossary. - - Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing. - - Preserve correctly identified glossary names and aliases. A recognized glossary term may still be a mishearing of another term; correct that misidentification only when spoken similarity and surrounding context support it. - - Apart from that supported mishearing exception, do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. - - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - - If a segment includes categories, treat them as additional transcript context. - - Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary. - - Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement. - - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + Rules: + - Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction. + - The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English. + - Appropriate example: correcting "gestures" to "Jesters" can be valid if "Jesters" appears in the glossary and nearby context supports that inference. + - Inappropriate example: correcting "Lyra" to "Jesters" should be omitted because those words are not similar in spoken English, even if "Jesters" appears in the glossary. + - Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing. + - Preserve correctly identified glossary names and aliases. A recognized glossary term may still be a mishearing of another term; correct that misidentification only when spoken similarity and surrounding context support it. + - Apart from that supported mishearing exception, do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. + - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. + - If a segment includes categories, treat them as additional transcript context. + - Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary. + - Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. - {{ .ProposalContract }} + {{ .ProposalContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Glossary: - {{ .GlossaryJSON }} + {{ .TranscriptDescriptionBlock }}Glossary: + {{ .GlossaryJSON }} - Transcript section: - {{ .SectionJSON }} - output: - format: json - validation_mode: json_schema - schema_path: correction_set.v1.json + Transcript section: + {{ .SectionJSON }} + output: + format: json + validation_mode: json_schema + schema_path: correction_set.v1.json diff --git a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml index e7bac8a..02da0b4 100644 --- a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml @@ -1,41 +1,42 @@ -prompts: - modules.grammar.proposal: - v1: - messages: - - role: system - content: | - You are Audita, a conservative grammar cleanup assistant. Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. The only word-substitution exception is the whole-word article "a"/"an" correction described below; otherwise do not change content, substitute words, or rewrite the speaker's phrasing. - - role: user - content: | - Review this transcript section and return only grammar cleanup corrections that should be applied. +collections: + audita: + prompts: + - id: modules.grammar.proposal + messages: + - role: system + content: | + You are Audita, a conservative grammar cleanup assistant. Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. The only word-substitution exception is the whole-word article "a"/"an" correction described below; otherwise do not change content, substitute words, or rewrite the speaker's phrasing. + - role: user + content: | + Review this transcript section and return only grammar cleanup corrections that should be applied. - Rules: - - Allowed changes are punctuation, capitalization, spacing, and article cleanup only. - - You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same. - - You may change the whole-word article "a" to "an" or "an" to "a" when the surrounding text otherwise stays the same. - - Homophone, spoken-form, and mistranscription corrections are outside this module's scope; do not propose them here. - - Apart from the a/an article exception, do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites. - - Do not change one written word into a different written word, except for capitalization changes to the same letters and the a/an article exception. - - If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup. - - Treat glossary names and aliases as protected spellings and context. - - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. - - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - - If a segment includes categories, treat them as additional transcript context. - - Choose an original_text span that appears exactly once in the current segment text. - - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. - - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + Rules: + - Allowed changes are punctuation, capitalization, spacing, and article cleanup only. + - You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same. + - You may change the whole-word article "a" to "an" or "an" to "a" when the surrounding text otherwise stays the same. + - Homophone, spoken-form, and mistranscription corrections are outside this module's scope; do not propose them here. + - Apart from the a/an article exception, do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites. + - Do not change one written word into a different written word, except for capitalization changes to the same letters and the a/an article exception. + - If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup. + - Treat glossary names and aliases as protected spellings and context. + - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. + - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. + - If a segment includes categories, treat them as additional transcript context. + - Choose an original_text span that appears exactly once in the current segment text. + - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. + - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. - {{ .ProposalContract }} + {{ .ProposalContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Protected glossary/context: - {{ .GlossaryJSON }} + {{ .TranscriptDescriptionBlock }}Protected glossary/context: + {{ .GlossaryJSON }} - Transcript section: - {{ .SectionJSON }} - output: - format: json - validation_mode: json_schema - schema_path: correction_set.v1.json + Transcript section: + {{ .SectionJSON }} + output: + format: json + validation_mode: json_schema + schema_path: correction_set.v1.json diff --git a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml index 5395019..3003979 100644 --- a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml @@ -1,38 +1,39 @@ -prompts: - modules.homophones.proposal: - v1: - messages: - - role: system - content: | - You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content. - - role: user - content: | - Review this transcript section and return only homophone or spoken-form corrections that should be applied. +collections: + audita: + prompts: + - id: modules.homophones.proposal + messages: + - role: system + content: | + You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content. + - role: user + content: | + Review this transcript section and return only homophone or spoken-form corrections that should be applied. - Rules: - - Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken. - - Allow examples such as changing "dam" to "damn", "rank" to "Hrank", or "gestures" to "Jesters" when local context supports the correction. - - Reject unrelated substitutions like changing "Lyra" to "Jesters". - - Reject antonyms or reversals such as changing "visible" to "invisible". - - Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits. - - Treat glossary names and aliases as protected spellings and context. - - You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context. - - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - - If a segment includes categories, treat them as additional transcript context. - - Choose an original_text span that appears exactly once in the current segment text. - - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + Rules: + - Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken. + - Allow examples such as changing "dam" to "damn", "rank" to "Hrank", or "gestures" to "Jesters" when local context supports the correction. + - Reject unrelated substitutions like changing "Lyra" to "Jesters". + - Reject antonyms or reversals such as changing "visible" to "invisible". + - Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits. + - Treat glossary names and aliases as protected spellings and context. + - You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context. + - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. + - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. + - If a segment includes categories, treat them as additional transcript context. + - Choose an original_text span that appears exactly once in the current segment text. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. - {{ .ProposalContract }} + {{ .ProposalContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Protected glossary/context: - {{ .GlossaryJSON }} + {{ .TranscriptDescriptionBlock }}Protected glossary/context: + {{ .GlossaryJSON }} - Transcript section: - {{ .SectionJSON }} - output: - format: json - validation_mode: json_schema - schema_path: correction_set.v1.json + Transcript section: + {{ .SectionJSON }} + output: + format: json + validation_mode: json_schema + schema_path: correction_set.v1.json diff --git a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml index 3205625..e941b83 100644 --- a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml @@ -1,43 +1,44 @@ -prompts: - modules.spoken_word.proposal: - v1: - messages: - - role: system - content: | - You are Audita, a conservative spoken-word cleanup assistant. Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, and similar dysfluencies that commonly appear in spoken English transcripts. Preserve substantive meaning, named entities, and transcript content. - - role: user - content: | - Review this transcript section and return only spoken-word cleanup corrections that should be applied. +collections: + audita: + prompts: + - id: modules.spoken_word.proposal + messages: + - role: system + content: | + You are Audita, a conservative spoken-word cleanup assistant. Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, and similar dysfluencies that commonly appear in spoken English transcripts. Preserve substantive meaning, named entities, and transcript content. + - role: user + content: | + Review this transcript section and return only spoken-word cleanup corrections that should be applied. - Rules: - - Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies. - - You may collapse adjacent repetition such as "I I think" to "I think" or remove filler spans such as "you know" or "uh" when local context supports that cleanup. - - Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency. - - Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved. - - Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart. - - You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup. - - Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits. - - Do not change clear content words just because a different phrasing reads better. - - Do not convert uncertain statements into certain statements. - - Treat glossary names and aliases as protected spellings and context. Accidental repetition of a protected name may be reduced when that identity remains in the segment; preserve substantive references and intentional emphasis. - - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - - If a segment includes categories, treat them as additional transcript context. - - Choose an original_text span that appears exactly once in the current segment text. - - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. - - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + Rules: + - Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies. + - You may collapse adjacent repetition such as "I I think" to "I think" or remove filler spans such as "you know" or "uh" when local context supports that cleanup. + - Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency. + - Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved. + - Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart. + - You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup. + - Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits. + - Do not change clear content words just because a different phrasing reads better. + - Do not convert uncertain statements into certain statements. + - Treat glossary names and aliases as protected spellings and context. Accidental repetition of a protected name may be reduced when that identity remains in the segment; preserve substantive references and intentional emphasis. + - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. + - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. + - If a segment includes categories, treat them as additional transcript context. + - Choose an original_text span that appears exactly once in the current segment text. + - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. + - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. - {{ .ProposalContract }} + {{ .ProposalContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Protected glossary/context: - {{ .GlossaryJSON }} + {{ .TranscriptDescriptionBlock }}Protected glossary/context: + {{ .GlossaryJSON }} - Transcript section: - {{ .SectionJSON }} - output: - format: json - validation_mode: json_schema - schema_path: correction_set.v1.json + Transcript section: + {{ .SectionJSON }} + output: + format: json + validation_mode: json_schema + schema_path: correction_set.v1.json diff --git a/internal/prompts/assets/prompts/validators.editorial_review.yaml b/internal/prompts/assets/prompts/validators.editorial_review.yaml index bafad62..9df9cea 100644 --- a/internal/prompts/assets/prompts/validators.editorial_review.yaml +++ b/internal/prompts/assets/prompts/validators.editorial_review.yaml @@ -1,35 +1,36 @@ -prompts: - validators.editorial_review: - v1: - messages: - - role: system - content: | - You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup. - - role: user - content: | - Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision. +collections: + audita: + prompts: + - id: validators.editorial_review + messages: + - role: system + content: | + You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup. + - role: user + content: | + Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision. - Rules: - - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}. - - Approve editorial revisions that preserve the underlying meaning of the segment. - - Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies. - - Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning. - - Approve low-risk homophone or mistranscription corrections when they are contextually well supported. - - Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving. - - Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency. - - Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved. - - Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart. - - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. - - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. + Rules: + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}. + - Approve editorial revisions that preserve the underlying meaning of the segment. + - Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies. + - Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning. + - Approve low-risk homophone or mistranscription corrections when they are contextually well supported. + - Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving. + - Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency. + - Phrases such as "Help! Help! Help!", "Stop! Stop! Stop!", "No! No! No!", "Yes! Yes! Yes!", and "Go! Go! Go!" are often intentional emphasis and should usually be preserved. + - Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart. + - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. + - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. + - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. - {{ .ValidationContract }} + {{ .ValidationContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Corrections to validate: - {{ .PayloadJSON }} - output: - format: json - validation_mode: json_schema - schema_path: validator_decision_set.v1.json + {{ .TranscriptDescriptionBlock }}Corrections to validate: + {{ .PayloadJSON }} + output: + format: json + validation_mode: json_schema + schema_path: validator_decision_set.v1.json diff --git a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml index 9835b43..7291dbb 100644 --- a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml +++ b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml @@ -1,31 +1,32 @@ -prompts: - validators.meaning_reversal_review: - v1: - messages: - - role: system - content: | - You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals. - - role: user - content: | - Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning. +collections: + audita: + prompts: + - id: validators.meaning_reversal_review + messages: + - role: system + content: | + You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals. + - role: user + content: | + Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning. - Rules: - - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}. - - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. - - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. - - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - - An empty corrected_text deletes the identified span; decide from the full before/after segment whether that deletion reverses meaning, not from the empty string alone. - - For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone. - - Do not reject a correction merely because the literal written word changes. - - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. + Rules: + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}. + - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. + - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. + - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. + - An empty corrected_text deletes the identified span; decide from the full before/after segment whether that deletion reverses meaning, not from the empty string alone. + - For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone. + - Do not reject a correction merely because the literal written word changes. + - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. - {{ .ValidationContract }} + {{ .ValidationContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Corrections to validate: - {{ .PayloadJSON }} - output: - format: json - validation_mode: json_schema - schema_path: validator_decision_set.v1.json + {{ .TranscriptDescriptionBlock }}Corrections to validate: + {{ .PayloadJSON }} + output: + format: json + validation_mode: json_schema + schema_path: validator_decision_set.v1.json diff --git a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml index e199c88..514977a 100644 --- a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml +++ b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml @@ -1,30 +1,31 @@ -prompts: - validators.spoken_form_plausibility: - v1: - messages: - - role: system - content: | - You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken. - - role: user - content: | - Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction. +collections: + audita: + prompts: + - id: validators.spoken_form_plausibility + messages: + - role: system + content: | + You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken. + - role: user + content: | + Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction. - Rules: - - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}. - - Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction. - - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". - - Reject unrelated substitutions like changing "Lyra" to "Jesters". - - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. - - For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. - - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. + Rules: + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}. + - Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction. + - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". + - Reject unrelated substitutions like changing "Lyra" to "Jesters". + - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. + - For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. + - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. - {{ .ValidationContract }} + {{ .ValidationContract }} - {{ .HardeningPolicy }} + {{ .HardeningPolicy }} - {{ .TranscriptDescriptionBlock }}Corrections to validate: - {{ .PayloadJSON }} - output: - format: json - validation_mode: json_schema - schema_path: validator_decision_set.v1.json + {{ .TranscriptDescriptionBlock }}Corrections to validate: + {{ .PayloadJSON }} + output: + format: json + validation_mode: json_schema + schema_path: validator_decision_set.v1.json diff --git a/internal/prompts/registry.go b/internal/prompts/registry.go index 9f2f07a..84f66ed 100644 --- a/internal/prompts/registry.go +++ b/internal/prompts/registry.go @@ -13,15 +13,14 @@ var embeddedAssets embed.FS const ( SourceBuiltin = "builtin" - VersionV1 = "v1" AssetRoot = "assets/prompts" - PromptIDModuleGlossaryProposal = "modules.glossary.proposal" - PromptIDModuleHomophonesProposal = "modules.homophones.proposal" - PromptIDModuleSpokenWordProposal = "modules.spoken_word.proposal" - PromptIDModuleGrammarProposal = "modules.grammar.proposal" - PromptIDValidatorSpokenFormPlausibility = "validators.spoken_form_plausibility" - PromptIDValidatorMeaningReversalReview = "validators.meaning_reversal_review" - PromptIDValidatorEditorialReview = "validators.editorial_review" + PromptIDModuleGlossaryProposal = "audita/modules.glossary.proposal" + PromptIDModuleHomophonesProposal = "audita/modules.homophones.proposal" + PromptIDModuleSpokenWordProposal = "audita/modules.spoken_word.proposal" + PromptIDModuleGrammarProposal = "audita/modules.grammar.proposal" + PromptIDValidatorSpokenFormPlausibility = "audita/validators.spoken_form_plausibility" + PromptIDValidatorMeaningReversalReview = "audita/validators.meaning_reversal_review" + PromptIDValidatorEditorialReview = "audita/validators.editorial_review" ) var promptIDs = []string{ @@ -31,15 +30,14 @@ var promptIDs = []string{ } type Metadata struct { - PromptID string `json:"prompt_id"` - PromptVersion string `json:"prompt_version"` - PromptSource string `json:"prompt_source"` - EmbeddedPath string `json:"embedded_path"` + PromptID string `json:"prompt_id"` + PromptSource string `json:"prompt_source"` + EmbeddedPath string `json:"embedded_path"` } func (m Metadata) DiagnosticsMap() map[string]any { return map[string]any{ - "prompt_id": m.PromptID, "prompt_version": m.PromptVersion, + "prompt_id": m.PromptID, "prompt_source": m.PromptSource, "embedded_path": m.EmbeddedPath, } } @@ -51,7 +49,7 @@ func LookupMetadata(promptID string) (Metadata, bool) { id := strings.TrimSpace(promptID) for _, known := range promptIDs { if known == id { - return Metadata{PromptID: id, PromptVersion: VersionV1, PromptSource: SourceBuiltin, EmbeddedPath: AssetRoot + "/" + id + ".yaml"}, true + return Metadata{PromptID: id, PromptSource: SourceBuiltin, EmbeddedPath: AssetRoot + "/" + strings.TrimPrefix(id, "audita/") + ".yaml"}, true } } return Metadata{}, false diff --git a/internal/prompts/registry_test.go b/internal/prompts/registry_test.go index cebd56b..239b8b2 100644 --- a/internal/prompts/registry_test.go +++ b/internal/prompts/registry_test.go @@ -5,9 +5,9 @@ import ( "strings" "testing" - ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" - pk "gitea.maximumdirect.net/eric/promptkit" + ck "go.maximumdirect.net/configkit" + pk "go.maximumdirect.net/promptkit" ) func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { @@ -20,8 +20,8 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { sources := pk.ConfigurationSources{ Underlays: []ck.Underlay{ {Inputs: promptInputs}, - {Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", map[string]pk.ProfileConfig{ - "fixture": {Endpoint: &endpoint, Model: &model}, + {Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", []pk.ProfileConfig{ + {ID: "fixture", Endpoint: &endpoint, Model: &model}, })}}}, }, } @@ -51,11 +51,11 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { {PromptIDValidatorEditorialReview, "editorial validation assistant", "validator_decision_set.v1.json", "fixture-correction"}, } { t.Run(tc.id, func(t *testing.T) { - prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: vars}) + prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: vars}) if err != nil { t.Fatal(err) } - if prepared.PromptID != tc.id || prepared.PromptVersion != VersionV1 || prepared.OutputContract.SchemaPath != tc.schema || prepared.StructuredOutput == nil { + if prepared.PromptID != tc.id || prepared.OutputContract.SchemaPath != tc.schema || prepared.StructuredOutput == nil { t.Fatalf("selection/schema: %+v", prepared) } if len(prepared.Messages) != 2 { @@ -67,11 +67,11 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { t.Errorf("missing %q", fragment) } } - if strings.HasPrefix(tc.id, "modules.") && !strings.Contains(rendered, "fixture-glossary") { + if strings.HasPrefix(tc.id, "audita/modules.") && !strings.Contains(rendered, "fixture-glossary") { t.Error("missing glossary context") } contract := "ValidationContract" - if strings.HasPrefix(tc.id, "modules.") { + if strings.HasPrefix(tc.id, "audita/modules.") { contract = "ProposalContract" } for _, line := range strings.Split(strings.TrimSpace(vars[contract]), "\n") { @@ -89,7 +89,7 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { } } missing := map[string]string{"HardeningPolicy": HardeningText()} - if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: missing}); err == nil { + if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: missing}); err == nil { t.Error("missing variables accepted") } }) @@ -102,7 +102,7 @@ func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) { t.Fatalf("prompt count: %d", len(all)) } for _, meta := range all { - if meta.PromptVersion != VersionV1 || meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" { + if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" { t.Fatalf("metadata: %+v", meta) } if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil {