package promptkit_test import ( "context" "encoding/json" "io" "net/http" "reflect" "strings" "testing" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) func TestBundledSmokeHTTPDefaults(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "synthetic-key") var urls []string var models []string transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { urls = append(urls, request.URL.String()) body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}` if request.Method == http.MethodPost { var payload map[string]any if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { t.Fatal(err) } models = append(models, payload["model"].(string)) content := "PROMPTKIT_OK" if _, structured := payload["response_format"]; structured { content = `{"status":"ok"}` } encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}}) if err != nil { t.Fatal(err) } body = string(encoded) } else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" { t.Errorf("authentication header missing") } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil }) engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}}) if err != nil { t.Fatal(err) } before, _ := engine.ListProfiles(context.Background()) inspectionBefore, err := engine.InspectConfiguration() if err != nil { t.Fatal(err) } for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} { report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"}) if err != nil || report == nil || !report.Completed || report.Failure != "" { t.Fatalf("%s report = %+v, %v", kind, report, err) } if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) { t.Fatalf("authentication facts = %+v", report.Facts) } if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput { if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" { t.Fatalf("target = %+v", report.Target) } } } wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"} if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) { t.Fatalf("urls %v, models %v", urls, models) } after, _ := engine.ListProfiles(context.Background()) if !reflect.DeepEqual(before, after) { t.Fatalf("inventory changed: %v %v", before, after) } inspectionAfter, err := engine.InspectConfiguration() if err != nil { t.Fatal(err) } diff, err := inspectionBefore.Diff(inspectionAfter) if err != nil || len(diff) != 0 { t.Fatalf("inspection changed: %+v, %v", diff, err) } } func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) { replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})} replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}} replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient)) if err != nil { t.Fatal(err) } replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"}) if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" { t.Fatalf("replaced default = %+v, %v", replacementReport, err) } replacementClient.response.Content = "unexpected" mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"}) if err != nil || mismatch == nil || !mismatch.Completed { t.Fatalf("concluded mismatch = %+v, %v", mismatch, err) } input := configkit.Input{Source: configkit.Mapping("application", map[string]any{ "backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}}, "profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, })} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}} engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } for _, test := range []struct { request promptkit.SmokeTestRequest profile, model string }{ {promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"}, {promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"}, {promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"}, } { test.request.Kind = promptkit.SmokeTestTextInference report, err := engine.SmokeTest(context.Background(), test.request) if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model { t.Fatalf("report = %+v, %v", report, err) } if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") { t.Fatal("application content used by diagnostic") } } }