package cli import ( "bytes" "context" "encoding/json" "io" "net/http" "os" "path/filepath" "runtime" "strings" "sync" "sync/atomic" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" ck "go.maximumdirect.net/configkit" ) func TestMaintainedProjectsWithControlledProvider(t *testing.T) { _, file, _, ok := runtime.Caller(0) if !ok { t.Fatal("cannot locate examples") } root := filepath.Join(filepath.Dir(file), "..", "..", "examples") t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key") for _, tc := range []struct{ dir, main, overlay string }{ {"local", "config.yml", ""}, {"service", "config.yml", "testing"}, {"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""}, } { t.Run(tc.dir+"/"+tc.main, func(t *testing.T) { main := filepath.Join(root, tc.dir, tc.main) loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay) if err != nil { t.Fatal(err) } loaded.Config.Reporting.Enabled = false var calls atomic.Int32 var mu sync.Mutex var models, prompts []string transport := controlledTransport(func(r *http.Request) (*http.Response, error) { calls.Add(1) var request struct { Model string `json:"model"` Messages []struct { Content string `json:"content"` } `json:"messages"` } if err := json.NewDecoder(r.Body).Decode(&request); err != nil { return nil, err } var content strings.Builder for _, message := range request.Messages { content.WriteString(message.Content) } mu.Lock() models = append(models, request.Model) prompts = append(prompts, content.String()) mu.Unlock() return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil }) engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } if tc.dir == "prompt-edit" { prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal") if err != nil || !strings.Contains(prompt.Messages[0].Content, "clearly supported grammatical errors") { t.Fatalf("operator prompt: %+v %v", prompt, err) } } raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json")) if err != nil { t.Fatal(err) } source, err := schema.ParseSourceTranscriptJSON(raw) if err != nil { t.Fatal(err) } inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source} prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{}) if err != nil { t.Fatal(err) } defer prepared.Prepared.Output.Close() if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil { t.Fatal(err) } if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) { t.Fatalf("provider calls=%d", calls.Load()) } if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") { t.Fatalf("overlay models=%v", models) } if tc.dir == "campaigns" && tc.main == "first.yml" { if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) { t.Fatalf("campaign glossary selections: %v", prompts) } } if tc.dir == "campaigns" && tc.main == "second.yml" { if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) { t.Fatalf("second campaign glossary selection: %v", prompts) } } }) } } func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) { _, file, _, ok := runtime.Caller(0) if !ok { t.Fatal("cannot locate examples") } source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit") dir := t.TempDir() for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} { raw, err := os.ReadFile(filepath.Join(source, name)) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil { t.Fatal(err) } } inspect := func() string { loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") if err != nil { t.Fatal(err) } engine, err := llm.NewRuntime(context.Background(), loaded, nil) if err != nil { t.Fatal(err) } prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal") if err != nil { t.Fatal(err) } return prompt.Messages[0].Content } before := inspect() filePath := filepath.Join(dir, "prompts.yml") raw, err := os.ReadFile(filePath) if err != nil { t.Fatal(err) } if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil { t.Fatal(err) } after := inspect() if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") { t.Fatalf("before=%q after=%q", before, after) } }