100 lines
4.0 KiB
Go
100 lines
4.0 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestNestedErrorsRetainEachCauseWithoutDuplicatingDiagnostics(t *testing.T) {
|
|
cause := errors.New("independent-leaf-diagnostic")
|
|
for range 10 {
|
|
parent := newSimpleEngine(t, "parent", promptkit.WithLLMClient(&fakeLLMClient{err: cause}))
|
|
_, err := parent.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
|
if !errors.Is(err, cause) || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
|
t.Fatalf("lost owner or cause: %v", err)
|
|
}
|
|
if strings.Count(err.Error(), "independent-leaf-diagnostic") != 1 {
|
|
t.Fatalf("duplicated leaf: %v", err)
|
|
}
|
|
cause = err
|
|
}
|
|
}
|
|
|
|
type nestedErrorReader struct{ cause error }
|
|
|
|
func (r nestedErrorReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
|
return nil, r.cause
|
|
}
|
|
|
|
func TestNestedEngineErrorKeepsOwningOperation(t *testing.T) {
|
|
child := newSimpleEngine(t, "child")
|
|
_, childErr := child.Run(context.Background(), promptkit.RunRequest{PromptID: "missing"})
|
|
if !errors.Is(childErr, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("child error = %v", childErr)
|
|
}
|
|
client := &fakeLLMClient{err: childErr}
|
|
parent := newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
|
_, err := parent.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
|
if !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, childErr) || !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("generation error = %v", err)
|
|
}
|
|
|
|
readerParent := newSimpleEngine(t, "hello", promptkit.WithArtifactReader(nestedErrorReader{cause: childErr}))
|
|
_, err = readerParent.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"extra": promptkit.Inline("data")}})
|
|
if !errors.Is(err, promptkit.ErrArtifactLoad) || !errors.Is(err, childErr) || !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("artifact error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestObservedNestedErrorsKeepTheParentOperation(t *testing.T) {
|
|
child := newSimpleEngine(t, "child")
|
|
_, childErr := child.Run(context.Background(), promptkit.RunRequest{PromptID: "missing"})
|
|
for _, mode := range []string{"run", "prepared", "repair", "artifact"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
var events []promptkit.ExecutionEvent
|
|
req := promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) { events = append(events, e) }}
|
|
engine := newSimpleEngine(t, "parent", promptkit.WithLLMClient(&fakeLLMClient{err: childErr}))
|
|
want, owner := promptkit.ExecutionFailureGeneration, promptkit.ErrLLMGenerate
|
|
if mode == "artifact" {
|
|
engine = newSimpleEngine(t, "parent", promptkit.WithArtifactReader(nestedErrorReader{cause: childErr}))
|
|
req.Inputs = map[string]promptkit.ArtifactRef{"extra": promptkit.Inline("data")}
|
|
want, owner = promptkit.ExecutionFailureArtifactLoad, promptkit.ErrArtifactLoad
|
|
}
|
|
if mode == "repair" {
|
|
calls := 0
|
|
engine = newSimpleEngine(t, "parent", promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
return &promptkit.GenerateResponse{Content: "invalid JSON"}, nil
|
|
}
|
|
return nil, childErr
|
|
})))
|
|
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
|
|
}
|
|
var err error
|
|
if mode == "prepared" {
|
|
handle, prepErr := engine.PrepareExecution(context.Background(), req)
|
|
if prepErr != nil {
|
|
t.Fatal(prepErr)
|
|
}
|
|
_, err = engine.RunPrepared(context.Background(), handle)
|
|
} else {
|
|
_, err = engine.Run(context.Background(), req)
|
|
}
|
|
if !errors.Is(err, owner) || !errors.Is(err, childErr) {
|
|
t.Fatalf("lost owner/cause: %v", err)
|
|
}
|
|
if len(events) == 0 || events[len(events)-1].Failure != want {
|
|
t.Fatalf("terminal classification: %+v", events)
|
|
}
|
|
if mode != "artifact" && events[len(events)-2].Failure != want {
|
|
t.Fatalf("attempt classification: %+v", events)
|
|
}
|
|
})
|
|
}
|
|
}
|