Keep long fallback reasons from invalidating warnings
This commit is contained in:
@@ -235,6 +235,10 @@ do not invent a generic identity framework for families that do not need one.
|
||||
|
||||
## Finding 4: Bound Framework-Generated Fallback Diagnostic Messages
|
||||
|
||||
**Status: Complete.** Framework fallback/cache samples share bounded UTF-8
|
||||
rendering without changing full outcome evidence or module diagnostic checks.
|
||||
Multibyte/invalid-UTF-8 regressions and a typed spell CLI fallback probe pass.
|
||||
|
||||
Priority: P2. Relevant stages: 9, 11, 24, 29, and 34.
|
||||
|
||||
### Evidence And Impact
|
||||
|
||||
69
internal/cli/fallback_diagnostic_test.go
Normal file
69
internal/cli/fallback_diagnostic_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type longProviderFailure struct{}
|
||||
|
||||
func (longProviderFailure) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, fmt.Errorf("provider failed: %s", strings.Repeat("a detailed upstream failure; ", 250))
|
||||
}
|
||||
|
||||
func TestPermissiveFallbackSurvivesLongNativeCause(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json")
|
||||
configuration := fmt.Sprintf(`version: 6
|
||||
output: {directory: %q}
|
||||
cache_directory: %q
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: audit
|
||||
endpoint: https://example.invalid/v1
|
||||
model: audit
|
||||
api_key_required: false
|
||||
pipelines:
|
||||
audit:
|
||||
llm_profile: audit
|
||||
mode: permissive
|
||||
max_additional_turns: 0
|
||||
input: {module: seriatim}
|
||||
artifacts:
|
||||
spells:
|
||||
extract: {module: dnd/spells}
|
||||
normalize: {module: dnd/spells}
|
||||
`, filepath.Join(directory, "output"), filepath.Join(directory, "cache"))
|
||||
if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(inputPath, []byte(`{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
status := RunWithOptions([]string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--cache", "bypass", "--llm-transient-retries=0", "--format", "json", "--progress=false"}, &stdout, &stderr, Options{LLMClient: longProviderFailure{}})
|
||||
if status != 0 {
|
||||
t.Fatalf("supported empty fallback failed: status=%d stderr=%s", status, stderr.String())
|
||||
}
|
||||
var report struct {
|
||||
FallbackCount int `json:"fallback_count"`
|
||||
Directory string `json:"output_directory"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.FallbackCount != 1 {
|
||||
t.Fatalf("fallback count = %d", report.FallbackCount)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(report.Directory, "lanes", "spells.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,27 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdiagnostics "gitea.maximumdirect.net/eric/notarius/internal/framework/diagnostics"
|
||||
)
|
||||
|
||||
// boundedDiagnosticMessage is for framework-authored samples only. Module
|
||||
// diagnostics still pass through ordinary contract validation unchanged.
|
||||
func boundedDiagnosticMessage(message string) string {
|
||||
message = strings.ToValidUTF8(message, "<22>")
|
||||
if len(message) > contracts.MaxDiagnosticMessageBytes {
|
||||
message = message[:contracts.MaxDiagnosticMessageBytes]
|
||||
for !utf8.ValidString(message) {
|
||||
message = message[:len(message)-1]
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func promoteProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
|
||||
if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil {
|
||||
return nil, err
|
||||
|
||||
37
internal/framework/pipeline/fallback_diagnostic_test.go
Normal file
37
internal/framework/pipeline/fallback_diagnostic_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestFallbackDiagnosticBoundsOnlyThePresentedReason(t *testing.T) {
|
||||
for _, reason := range []string{
|
||||
strings.Repeat("a", contracts.MaxDiagnosticMessageBytes-1) + "界界",
|
||||
"provider error: " + string([]byte{0xff}) + strings.Repeat("x", contracts.MaxDiagnosticMessageBytes),
|
||||
} {
|
||||
outcome := contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: reason}
|
||||
groups, err := operationDiagnosticGroups(nil, nil, contracts.CandidateAssessment{}, outcome, contracts.DiagnosticOrigin{Stage: "extract", ModuleKey: "test"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(groups) != 1 || len(groups[0].Samples) != 1 {
|
||||
t.Fatalf("missing fallback warning: %#v", groups)
|
||||
}
|
||||
message := groups[0].Samples[0].Message
|
||||
if strings.TrimSpace(message) == "" || len(message) > contracts.MaxDiagnosticMessageBytes || !utf8.ValidString(message) {
|
||||
t.Fatalf("invalid warning sample: %q", message)
|
||||
}
|
||||
if outcome.FallbackReason != reason {
|
||||
t.Fatal("original failure reason was changed")
|
||||
}
|
||||
invalid := []contracts.ProducerDiagnostic{{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback,
|
||||
ReasonCode: "module_warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "test", Message: reason}}}}
|
||||
if _, err := operationDiagnosticGroups(invalid, nil, contracts.CandidateAssessment{}, outcome, contracts.DiagnosticOrigin{Stage: "extract", ModuleKey: "test"}, nil); err == nil {
|
||||
t.Fatal("invalid module diagnostic was silently repaired")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,6 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -47,13 +45,7 @@ func producerCacheObserver(input RunInput, origin contracts.DiagnosticOrigin, ch
|
||||
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
|
||||
}
|
||||
if observation.Err != nil {
|
||||
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
|
||||
if len(message) > contracts.MaxDiagnosticMessageBytes {
|
||||
message = message[:contracts.MaxDiagnosticMessageBytes]
|
||||
for !utf8.ValidString(message) {
|
||||
message = message[:len(message)-1]
|
||||
}
|
||||
}
|
||||
message := boundedDiagnosticMessage(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err))
|
||||
*warnings = append(*warnings, contracts.ProducerDiagnostic{
|
||||
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
|
||||
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
|
||||
|
||||
@@ -52,7 +52,7 @@ func operationDiagnosticGroups(diagnostics, cacheDiagnostics []contracts.Produce
|
||||
warning := contracts.ProducerDiagnostic{
|
||||
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback,
|
||||
ReasonCode: "producer_fallback", OccurrenceCount: 1,
|
||||
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: outcome.FallbackReason}},
|
||||
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: boundedDiagnosticMessage(outcome.FallbackReason)}},
|
||||
}
|
||||
fallbackGroups, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{warning}, origin, chunk)
|
||||
return append(groups, fallbackGroups...), err
|
||||
|
||||
Reference in New Issue
Block a user