diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0cf2705..19ea49e 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–9 complete; Stage 10 is next. +Status: Stages 1–10 complete; Stage 11 is next. ## Authority and baseline @@ -533,6 +533,16 @@ serialized; do not introduce filtering of returned `ExtraParams`. **Complete when:** native smoke evidence and aggregate entries have a complete, neutral YAML representation. +**Completed:** 2026-10-17. Added private report and aggregate DTOs that explicitly +project every exported PromptKit smoke report field, including target settings, +presence, nullable facts, timing, attempts, usage, and failure details. Durations +render as Go duration strings, empty attempts as `[]`, and optional prompt +versions preserve empty values while omitting absent ones. Aggregate entries +retain order, nullable reports, native report kinds, and optional error text. +Semantic YAML tests cover all field categories, nested values, omission, +presence, direct-request/key exclusion, and report-plus-error entries. Focused +projection tests and `GOTOOLCHAIN=local go build ./...` passed. + ## Stage 11: Bind and execute single smoke tests **Primary files:** private `smoke_command.go`, `Command` assembly, basic command diff --git a/integrations/ckprompt/smoke_report.go b/integrations/ckprompt/smoke_report.go new file mode 100644 index 0000000..c9e98e6 --- /dev/null +++ b/integrations/ckprompt/smoke_report.go @@ -0,0 +1,234 @@ +package ckprompt + +import ( + "time" + + promptkit "gitea.maximumdirect.net/eric/promptkit" +) + +type smokeReportDTO struct { + Kind promptkit.SmokeTestKind `yaml:"kind"` + Supported bool `yaml:"supported"` + Applicable *bool `yaml:"applicable"` + Completed bool `yaml:"completed"` + Reason promptkit.SmokeTestReason `yaml:"reason"` + Target smokeTargetDTO `yaml:"target"` + Facts smokeFactsDTO `yaml:"facts"` + Timing smokeTimingDTO `yaml:"timing"` + Attempts []smokeAttemptDTO `yaml:"attempts"` + Usage smokeUsageDTO `yaml:"usage"` + Failure promptkit.ExecutionFailureKind `yaml:"failure"` +} + +type smokeTargetDTO struct { + BackendID string `yaml:"backend_id"` + ProfileID string `yaml:"profile_id"` + PromptID string `yaml:"prompt_id"` + PromptVersion *string `yaml:"version,omitempty"` + DefaultSourceBackendID string `yaml:"default_source_backend_id"` + SelectionSource string `yaml:"selection_source"` + Endpoint string `yaml:"endpoint"` + EffectiveSettings smokeExecutionTargetDTO `yaml:"effective_settings"` + SettingsPresence smokeSettingsPresenceDTO `yaml:"settings_presence"` +} + +type smokeSettingsPresenceDTO struct { + Temperature bool `yaml:"temperature"` + MaxTokens bool `yaml:"max_tokens"` + TopP bool `yaml:"top_p"` + TimeoutSeconds bool `yaml:"timeout_seconds"` +} + +type smokeExecutionTargetDTO struct { + BackendID string `yaml:"backend_id,omitempty"` + Endpoint string `yaml:"endpoint"` + Model string `yaml:"model"` + Temperature float64 `yaml:"temperature"` + MaxTokens int `yaml:"max_tokens"` + TopP float64 `yaml:"top_p"` + TimeoutSeconds int `yaml:"timeout_seconds"` + ServiceTier string `yaml:"service_tier"` + ReasoningEffort string `yaml:"reasoning_effort"` + APIKeyEnv string `yaml:"api_key_env"` + ExtraParams map[string]any `yaml:"extra_params"` +} + +type smokeFactsDTO struct { + CredentialsSupplied *bool `yaml:"credentials_supplied"` + EndpointReachable *bool `yaml:"endpoint_reachable"` + CredentialsAccepted *bool `yaml:"credentials_accepted"` + ResponseReceived *bool `yaml:"response_received"` + GenerationCompleted *bool `yaml:"generation_completed"` + TextReturned *bool `yaml:"text_returned"` + ExpectedTextMatched *bool `yaml:"expected_text_matched"` + StructuredOutputRequested *bool `yaml:"structured_output_requested"` + ValidJSON *bool `yaml:"valid_json"` + SchemaMatched *bool `yaml:"schema_matched"` +} + +type smokeTimingDTO struct { + StartedAt time.Time `yaml:"started_at"` + Duration string `yaml:"duration"` + CapacityWait *string `yaml:"capacity_wait"` + RequestDuration *string `yaml:"request_duration"` + RetryDelay *string `yaml:"retry_delay"` + EvaluationDuration *string `yaml:"evaluation_duration"` +} + +type smokeAttemptDTO struct { + Dispatched bool `yaml:"dispatched"` + InitialEndpoint string `yaml:"initial_endpoint"` + Endpoint string `yaml:"endpoint"` + HTTPStatus *int `yaml:"http_status"` + Facts smokeFactsDTO `yaml:"facts"` + Timing smokeTimingDTO `yaml:"timing"` + Failure promptkit.ExecutionFailureKind `yaml:"failure"` + Usage smokeUsageDTO `yaml:"usage"` +} + +type smokeUsageDTO struct { + PromptTokens promptkit.ObservedTokenCount `yaml:"prompt_tokens"` + CompletionTokens promptkit.ObservedTokenCount `yaml:"completion_tokens"` + TotalTokens promptkit.ObservedTokenCount `yaml:"total_tokens"` + CachedTokens promptkit.ObservedTokenCount `yaml:"cached_tokens"` + CacheWriteTokens promptkit.ObservedTokenCount `yaml:"cache_write_tokens"` +} + +type smokeAggregateDTO struct { + Tests []smokeResultDTO `yaml:"tests"` +} + +type smokeResultDTO struct { + Kind promptkit.SmokeTestKind `yaml:"kind"` + Report *smokeReportDTO `yaml:"report"` + Error string `yaml:"error,omitempty"` +} + +type smokeResult struct { + Kind promptkit.SmokeTestKind + Report *promptkit.SmokeTestReport + Err error +} + +func encodeSmokeReport(report *promptkit.SmokeTestReport) ([]byte, error) { + return encodePromptYAMLBytes(smokeReportFromNative(report)) +} + +func encodeSmokeResults(results []smokeResult) ([]byte, error) { + entries := make([]smokeResultDTO, len(results)) + for index, result := range results { + entries[index] = smokeResultDTO{ + Kind: result.Kind, + Report: smokeReportFromNative(result.Report), + } + if result.Err != nil { + entries[index].Error = result.Err.Error() + } + } + return encodePromptYAMLBytes(smokeAggregateDTO{Tests: entries}) +} + +func smokeReportFromNative(report *promptkit.SmokeTestReport) *smokeReportDTO { + if report == nil { + return nil + } + attempts := make([]smokeAttemptDTO, len(report.Attempts)) + for index, attempt := range report.Attempts { + attempts[index] = smokeAttemptDTO{ + Dispatched: attempt.Dispatched, + InitialEndpoint: attempt.InitialEndpoint, + Endpoint: attempt.Endpoint, + HTTPStatus: attempt.HTTPStatus, + Facts: smokeFactsFromNative(attempt.Facts), + Timing: smokeTimingFromNative(attempt.Timing), + Failure: attempt.Failure, + Usage: smokeUsageFromNative(attempt.Usage), + } + } + target := report.Target + settings := target.EffectiveSettings + return &smokeReportDTO{ + Kind: report.Kind, + Supported: report.Supported, + Applicable: report.Applicable, + Completed: report.Completed, + Reason: report.Reason, + Target: smokeTargetDTO{ + BackendID: target.BackendID, + ProfileID: target.ProfileID, + PromptID: target.PromptID, + PromptVersion: target.PromptVersion, + DefaultSourceBackendID: target.DefaultSourceBackendID, + SelectionSource: target.SelectionSource, + Endpoint: target.Endpoint, + EffectiveSettings: smokeExecutionTargetDTO{ + BackendID: settings.BackendID, + Endpoint: settings.Endpoint, + Model: settings.Model, + Temperature: settings.Temperature, + MaxTokens: settings.MaxTokens, + TopP: settings.TopP, + TimeoutSeconds: settings.TimeoutSeconds, + ServiceTier: settings.ServiceTier, + ReasoningEffort: settings.ReasoningEffort, + APIKeyEnv: settings.APIKeyEnv, + ExtraParams: settings.ExtraParams, + }, + SettingsPresence: smokeSettingsPresenceDTO{ + Temperature: target.SettingsPresence.Temperature, + MaxTokens: target.SettingsPresence.MaxTokens, + TopP: target.SettingsPresence.TopP, + TimeoutSeconds: target.SettingsPresence.TimeoutSeconds, + }, + }, + Facts: smokeFactsFromNative(report.Facts), + Timing: smokeTimingFromNative(report.Timing), + Attempts: attempts, + Usage: smokeUsageFromNative(report.Usage), + Failure: report.Failure, + } +} + +func smokeFactsFromNative(facts promptkit.SmokeTestFacts) smokeFactsDTO { + return smokeFactsDTO{ + CredentialsSupplied: facts.CredentialsSupplied, + EndpointReachable: facts.EndpointReachable, + CredentialsAccepted: facts.CredentialsAccepted, + ResponseReceived: facts.ResponseReceived, + GenerationCompleted: facts.GenerationCompleted, + TextReturned: facts.TextReturned, + ExpectedTextMatched: facts.ExpectedTextMatched, + StructuredOutputRequested: facts.StructuredOutputRequested, + ValidJSON: facts.ValidJSON, + SchemaMatched: facts.SchemaMatched, + } +} + +func smokeTimingFromNative(timing promptkit.SmokeTestTiming) smokeTimingDTO { + return smokeTimingDTO{ + StartedAt: timing.StartedAt, + Duration: timing.Duration.String(), + CapacityWait: durationString(timing.CapacityWait), + RequestDuration: durationString(timing.RequestDuration), + RetryDelay: durationString(timing.RetryDelay), + EvaluationDuration: durationString(timing.EvaluationDuration), + } +} + +func durationString(duration *time.Duration) *string { + if duration == nil { + return nil + } + value := duration.String() + return &value +} + +func smokeUsageFromNative(usage promptkit.ObservedTokenUsage) smokeUsageDTO { + return smokeUsageDTO{ + PromptTokens: usage.PromptTokens, + CompletionTokens: usage.CompletionTokens, + TotalTokens: usage.TotalTokens, + CachedTokens: usage.CachedTokens, + CacheWriteTokens: usage.CacheWriteTokens, + } +} diff --git a/integrations/ckprompt/smoke_report_test.go b/integrations/ckprompt/smoke_report_test.go new file mode 100644 index 0000000..de22598 --- /dev/null +++ b/integrations/ckprompt/smoke_report_test.go @@ -0,0 +1,211 @@ +package ckprompt + +import ( + "errors" + "testing" + "time" + + promptkit "gitea.maximumdirect.net/eric/promptkit" + "go.yaml.in/yaml/v4" +) + +func TestSmokeReportYAMLProjectsNativeFieldsAndPresence(t *testing.T) { + applicable := false + credentialsSupplied := false + endpointReachable := true + textReturned := false + zero := time.Duration(0) + retryDelay := 1500 * time.Millisecond + version := "" + httpStatus := 201 + started := time.Date(2026, time.October, 17, 12, 30, 0, 0, time.UTC) + report := &promptkit.SmokeTestReport{ + Kind: promptkit.SmokeTestReachability, + Supported: true, + Applicable: &applicable, + Completed: false, + Reason: promptkit.SmokeTestReasonHTTPDiagnosticsUnavailable, + Target: promptkit.SmokeTestTarget{ + BackendID: "local", + ProfileID: "default", + PromptID: "team/briefing", + PromptVersion: &version, + DefaultSourceBackendID: "fallback", + SelectionSource: "prompt", + Endpoint: "https://example.invalid/v1", + EffectiveSettings: promptkit.ExecutionTarget{ + BackendID: "local", + Endpoint: "https://example.invalid/v1", + Model: "example-model", + Temperature: 0, + MaxTokens: 0, + TopP: 0, + TimeoutSeconds: 0, + ServiceTier: "", + ReasoningEffort: "", + APIKeyEnv: "PROMPTKIT_KEY", + ExtraParams: map[string]any{"token": "caller-value", "nested": map[string]any{"count": 0}}, + }, + SettingsPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: false, TopP: true, TimeoutSeconds: false}, + }, + Facts: promptkit.SmokeTestFacts{ + CredentialsSupplied: &credentialsSupplied, + EndpointReachable: &endpointReachable, + CredentialsAccepted: nil, + ResponseReceived: &textReturned, + GenerationCompleted: &endpointReachable, + TextReturned: &textReturned, + ExpectedTextMatched: &textReturned, + StructuredOutputRequested: &endpointReachable, + ValidJSON: &textReturned, + SchemaMatched: nil, + }, + Timing: promptkit.SmokeTestTiming{ + StartedAt: started, + Duration: zero, + CapacityWait: &zero, + RequestDuration: nil, + RetryDelay: &retryDelay, + EvaluationDuration: nil, + }, + Attempts: []promptkit.SmokeTestAttempt{{ + Dispatched: true, + InitialEndpoint: "https://initial.example/v1", + Endpoint: "https://final.example/v1", + HTTPStatus: &httpStatus, + Facts: promptkit.SmokeTestFacts{EndpointReachable: &endpointReachable}, + Timing: promptkit.SmokeTestTiming{Duration: retryDelay}, + Failure: promptkit.ExecutionFailureNone, + Usage: promptkit.ObservedTokenUsage{PromptTokens: promptkit.ObservedTokenCount{Value: 2, Known: true, Complete: true}}, + }}, + Usage: promptkit.ObservedTokenUsage{ + PromptTokens: promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true}, + CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false}, + TotalTokens: promptkit.ObservedTokenCount{Value: 7, Known: true, Complete: false}, + }, + Failure: promptkit.ExecutionFailureTransport, + } + + encoded, err := encodeSmokeReport(report) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(encoded, &document); err != nil { + t.Fatalf("decode smoke report: %v\n%s", err, encoded) + } + if document["kind"] != "reachability" || document["supported"] != true || document["applicable"] != false || document["completed"] != false || document["reason"] != "http_diagnostics_unavailable" || document["failure"] != "transport" { + t.Fatalf("report scalar values = %#v", document) + } + target := document["target"].(map[string]any) + if target["prompt_id"] != "team/briefing" || target["version"] != "" || target["selection_source"] != "prompt" { + t.Fatalf("target values = %#v", target) + } + settings := target["effective_settings"].(map[string]any) + if settings["model"] != "example-model" || settings["temperature"] != 0 || settings["api_key_env"] != "PROMPTKIT_KEY" { + t.Fatalf("effective settings = %#v", settings) + } + extra := settings["extra_params"].(map[string]any) + if extra["token"] != "caller-value" { + t.Fatalf("extra parameters were not retained: %#v", extra) + } + presence := target["settings_presence"].(map[string]any) + if presence["temperature"] != true || presence["max_tokens"] != false || presence["top_p"] != true || presence["timeout_seconds"] != false { + t.Fatalf("setting presence = %#v", presence) + } + facts := document["facts"].(map[string]any) + if facts["credentials_supplied"] != false || facts["endpoint_reachable"] != true || facts["credentials_accepted"] != nil || facts["response_received"] != false || facts["generation_completed"] != true || facts["text_returned"] != false || facts["expected_text_matched"] != false || facts["structured_output_requested"] != true || facts["valid_json"] != false || facts["schema_matched"] != nil { + t.Fatalf("nullable facts = %#v", facts) + } + timing := document["timing"].(map[string]any) + if timing["started_at"] == nil || timing["duration"] != "0s" || timing["capacity_wait"] != "0s" || timing["request_duration"] != nil || timing["retry_delay"] != "1.5s" || timing["evaluation_duration"] != nil { + t.Fatalf("timing values = %#v", timing) + } + attempts, ok := document["attempts"].([]any) + if !ok || len(attempts) != 1 { + t.Fatalf("attempts = %#v", document["attempts"]) + } + attempt := attempts[0].(map[string]any) + if attempt["dispatched"] != true || attempt["initial_endpoint"] != "https://initial.example/v1" || attempt["endpoint"] != "https://final.example/v1" || attempt["http_status"] != 201 { + t.Fatalf("attempt fields = %#v", attempt) + } + if attempt["timing"].(map[string]any)["duration"] != "1.5s" { + t.Fatalf("attempt timing = %#v", attempt["timing"]) + } + usage := document["usage"].(map[string]any) + promptTokens := usage["prompt_tokens"].(map[string]any) + unknownTokens := usage["completion_tokens"].(map[string]any) + if promptTokens["value"] != 0 || promptTokens["known"] != true || promptTokens["complete"] != true || unknownTokens["known"] != false || unknownTokens["complete"] != false { + t.Fatalf("presence-aware usage = %#v", usage) + } + if containsSmokeKey(document, "request") || containsSmokeKey(document, "api_key") { + t.Fatalf("serialized request or direct API key: %#v", document) + } +} + +func TestSmokeReportYAMLEmitsNilAttemptsAsEmptySequence(t *testing.T) { + encoded, err := encodeSmokeReport(&promptkit.SmokeTestReport{Attempts: nil}) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(encoded, &document); err != nil { + t.Fatal(err) + } + if attempts, ok := document["attempts"].([]any); !ok || len(attempts) != 0 { + t.Fatalf("nil native attempts = %#v", document["attempts"]) + } +} + +func TestSmokeAggregateYAMLRetainsOrderErrorAndNativeReportKind(t *testing.T) { + operationErr := errors.New("operation failed") + report := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestReachability, Attempts: []promptkit.SmokeTestAttempt{}} + encoded, err := encodeSmokeResults([]smokeResult{ + {Kind: promptkit.SmokeTestTextInference, Report: report, Err: operationErr}, + {Kind: promptkit.SmokeTestAuthentication}, + }) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(encoded, &document); err != nil { + t.Fatalf("decode aggregate report: %v\n%s", err, encoded) + } + tests, ok := document["tests"].([]any) + if !ok || len(tests) != 2 { + t.Fatalf("aggregate entries = %#v", document) + } + first := tests[0].(map[string]any) + firstReport := first["report"].(map[string]any) + if first["kind"] != "text_inference" || firstReport["kind"] != "reachability" || first["error"] != "operation failed" { + t.Fatalf("first aggregate entry = %#v", first) + } + if _, exists := firstReport["target"].(map[string]any)["version"]; exists { + t.Fatalf("absent target version was emitted: %#v", firstReport["target"]) + } + second := tests[1].(map[string]any) + if second["kind"] != "authentication" || second["report"] != nil { + t.Fatalf("second aggregate entry = %#v", second) + } + if _, exists := second["error"]; exists { + t.Fatalf("nil error should be omitted: %#v", second) + } +} + +func containsSmokeKey(value any, want string) bool { + switch value := value.(type) { + case map[string]any: + for key, nested := range value { + if key == want || containsSmokeKey(nested, want) { + return true + } + } + case []any: + for _, nested := range value { + if containsSmokeKey(nested, want) { + return true + } + } + } + return false +}