Document the new configuration and processing workflow

internal/cli/examples_integration_test.go

@@ -0,0 +1,156 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
+	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
+	ck "go.maximumdirect.net/configkit"
+)
+
+func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("cannot locate examples")
+	}
+	root := filepath.Join(filepath.Dir(file), "..", "..", "examples")
+	t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key")
+	for _, tc := range []struct{ dir, main, overlay string }{
+		{"local", "config.yml", ""}, {"service", "config.yml", "testing"},
+		{"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""},
+	} {
+		t.Run(tc.dir+"/"+tc.main, func(t *testing.T) {
+			main := filepath.Join(root, tc.dir, tc.main)
+			loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay)
+			if err != nil {
+				t.Fatal(err)
+			}
+			loaded.Config.Reporting.Enabled = false
+			var calls atomic.Int32
+			var mu sync.Mutex
+			var models, prompts []string
+			transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
+				calls.Add(1)
+				var request struct {
+					Model    string `json:"model"`
+					Messages []struct {
+						Content string `json:"content"`
+					} `json:"messages"`
+				}
+				if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+					return nil, err
+				}
+				var content strings.Builder
+				for _, message := range request.Messages {
+					content.WriteString(message.Content)
+				}
+				mu.Lock()
+				models = append(models, request.Model)
+				prompts = append(prompts, content.String())
+				mu.Unlock()
+				return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil
+			})
+			engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tc.dir == "prompt-edit" {
+				prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
+				if err != nil || !strings.Contains(prompt.Messages[0].Content, "clearly supported grammatical errors") {
+					t.Fatalf("operator prompt: %+v %v", prompt, err)
+				}
+			}
+			raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			source, err := schema.ParseSourceTranscriptJSON(raw)
+			if err != nil {
+				t.Fatal(err)
+			}
+			inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source}
+			prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer prepared.Prepared.Output.Close()
+			if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil {
+				t.Fatal(err)
+			}
+			if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
+				t.Fatalf("provider calls=%d", calls.Load())
+			}
+			if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") {
+				t.Fatalf("overlay models=%v", models)
+			}
+			if tc.dir == "campaigns" && tc.main == "first.yml" {
+				if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) {
+					t.Fatalf("campaign glossary selections: %v", prompts)
+				}
+			}
+			if tc.dir == "campaigns" && tc.main == "second.yml" {
+				if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) {
+					t.Fatalf("second campaign glossary selection: %v", prompts)
+				}
+			}
+		})
+	}
+}
+
+func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("cannot locate examples")
+	}
+	source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit")
+	dir := t.TempDir()
+	for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} {
+		raw, err := os.ReadFile(filepath.Join(source, name))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	inspect := func() string {
+		loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		engine, err := llm.NewRuntime(context.Background(), loaded, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return prompt.Messages[0].Content
+	}
+	before := inspect()
+	filePath := filepath.Join(dir, "prompts.yml")
+	raw, err := os.ReadFile(filePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil {
+		t.Fatal(err)
+	}
+	after := inspect()
+	if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") {
+		t.Fatalf("before=%q after=%q", before, after)
+	}
+}