audita/examples/library-adoption/workflow_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strings"
	"sync"
	"testing"

	ck "gitea.maximumdirect.net/eric/ConfigKit"
	pk "gitea.maximumdirect.net/eric/promptkit"
)

func TestDemonstrationSelectsOutboundModels(t *testing.T) {
	for _, tc := range []struct {
		name                          string
		args                          []string
		overlay, proposal, assessment string
	}{
		{"base", nil, "", "synthetic-grammar", "synthetic-review"},
		{"testing", []string{"--overlay", "testing"}, "testing", "synthetic-grammar-testing", "synthetic-review-testing"},
		{"proposal override", []string{"--overlay", "testing", "--proposal-profile", "proposal"}, "testing", "synthetic-grammar", "synthetic-review-testing"},
		{"assessment override", []string{"--overlay", "testing", "--validation-profile", "assessment"}, "testing", "synthetic-grammar-testing", "synthetic-review"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
			var mu sync.Mutex
			var models []string
			base := demonstrationClient()
			client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
				body, err := io.ReadAll(req.Body)
				if err != nil {
					return nil, err
				}
				var wire struct {
					Model string `json:"model"`
				}
				if err := json.Unmarshal(body, &wire); err != nil {
					return nil, err
				}
				mu.Lock()
				models = append(models, wire.Model)
				mu.Unlock()
				req.Body = io.NopCloser(bytes.NewReader(body))
				return base.Transport.RoundTrip(req)
			})}
			var out bytes.Buffer
			if err := run(tc.args, &out, client); err != nil {
				t.Fatal(err)
			}
			var got struct {
				Overlay    string                                   `json:"overlay"`
				Proposal   struct{ Profile, Backend, Model string } `json:"proposal"`
				Assessment struct{ Profile, Backend, Model string } `json:"assessment"`
				Result     WorkflowResult                           `json:"result"`
				Attempts   uint64                                   `json:"attempts"`
				Retries    uint64                                   `json:"transient_retries"`
				Repairs    uint64                                   `json:"structural_repairs"`
			}
			if err := json.Unmarshal(out.Bytes(), &got); err != nil {
				t.Fatal(err)
			}
			if got.Overlay != tc.overlay || got.Proposal.Model != tc.proposal || got.Assessment.Model != tc.assessment || got.Proposal.Backend != "offline" || got.Assessment.Backend != "offline" {
				t.Fatalf("selection = %+v", got)
			}
			profiles := map[string]string{"synthetic-grammar": "proposal", "synthetic-review": "assessment", "synthetic-grammar-testing": "proposal-testing", "synthetic-review-testing": "assessment-testing"}
			if got.Proposal.Profile != profiles[tc.proposal] || got.Assessment.Profile != profiles[tc.assessment] {
				t.Fatalf("reported profiles = %q, %q", got.Proposal.Profile, got.Assessment.Profile)
			}
			if got.Result.Accepted != 1 || got.Attempts != 4 || got.Retries != 0 || got.Repairs != 2 {
				t.Fatalf("outcome = %+v", got)
			}
			mu.Lock()
			defer mu.Unlock()
			if len(models) != 4 || models[0] != tc.proposal || models[1] != tc.proposal || models[2] != tc.assessment || models[3] != tc.assessment {
				t.Fatalf("outbound models = %v", models)
			}
		})
	}
}

type step struct {
	status  int
	content string
	usage   any
}

func testRuntime(t *testing.T, transport http.RoundTripper, overrides ...ck.Override) *Runtime {
	t.Helper()
	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
	loaded, err := loadConfig(ck.File("config.yaml"), "", overrides...)
	if err != nil {
		t.Fatal(err)
	}
	r, err := newRuntime(loaded.Config, ".", &http.Client{Transport: transport})
	if err != nil {
		t.Fatal(err)
	}
	return r
}
func intOverride(field string, value int) ck.Override {
	return ck.Override{Path: ck.Keys("llm", field), Value: value, Flag: "--" + field}
}

func TestWorkflowBoundaries(t *testing.T) {
	zero := map[string]any{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
	for _, tc := range []struct {
		name                              string
		steps                             []step
		retries, repairs                  int
		accepted, rejected, warnings      int
		attempts, retryCount, repairCount uint64
		cause                             error
	}{
		{name: "repair both roles", steps: []step{{200, "bad", zero}, {200, validProposal, zero}, {200, "bad", nil}, {200, validAssessment, zero}}, repairs: 2, accepted: 1, attempts: 4, repairCount: 2},
		{name: "proposal repair exhausted", steps: []step{{200, "bad", nil}, {200, "bad", nil}, {200, "bad", nil}}, repairs: 2, warnings: 1, attempts: 3, repairCount: 2},
		{name: "assessment repair exhausted", steps: []step{{200, validProposal, nil}, {200, "bad", nil}, {200, "bad", nil}}, repairs: 1, rejected: 1, warnings: 1, attempts: 3, repairCount: 1},
		{name: "domain rejects missing span", steps: []step{{200, strings.Replace(validProposal, "hello world", "absent span", 1), nil}}, repairs: 2, rejected: 1, attempts: 1},
		{name: "domain rejects missing decision", steps: []step{{200, validProposal, nil}, {200, `{"validations":[]}`, nil}}, repairs: 2, rejected: 1, warnings: 1, attempts: 2},
		{name: "domain rejects duplicate decision", steps: []step{{200, validProposal, nil}, {200, `{"validations":[{"correction_index":0,"approved":true,"confidence":1,"reason":"ok"},{"correction_index":0,"approved":true,"confidence":1,"reason":"ok"}]}`, nil}}, repairs: 2, rejected: 1, warnings: 1, attempts: 2},
		{name: "assessment semantic rejection", steps: []step{{200, validProposal, nil}, {200, strings.Replace(validAssessment, `"approved":true`, `"approved":false`, 1), nil}}, repairs: 2, rejected: 1, attempts: 2},
		{name: "429 recovery", steps: []step{{429, "", nil}, {200, validProposal, zero}, {200, validAssessment, zero}}, retries: 1, accepted: 1, attempts: 3, retryCount: 1},
		{name: "explicit zero retries", steps: []step{{503, "", nil}}, repairs: 2, attempts: 1, cause: pk.ErrLLMGenerate},
		{name: "retry exhaustion after completed proposal", steps: []step{{200, validProposal, zero}, {503, "", nil}, {503, "", nil}}, retries: 1, repairs: 2, attempts: 3, retryCount: 1, cause: pk.ErrRetriesExhausted},
		{name: "retry and repair budgets", steps: []step{{503, "", nil}, {200, "bad", zero}, {503, "", nil}, {200, validProposal, zero}, {200, validAssessment, zero}}, retries: 1, repairs: 1, accepted: 1, attempts: 5, retryCount: 2, repairCount: 1},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var mu sync.Mutex
			count := 0
			transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
				mu.Lock()
				defer mu.Unlock()
				if req.Header.Get("Authorization") != "Bearer synthetic-test-secret" {
					t.Error("wrong credential")
				}
				var body struct {
					Messages []struct {
						Content string `json:"content"`
					} `json:"messages"`
				}
				if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
					return nil, err
				}
				if len(body.Messages) < 2 || !strings.Contains(body.Messages[1].Content, "Prompt hardening policy") {
					t.Error("real prompt hardening missing")
				}
				if count >= len(tc.steps) {
					t.Error("unexpected extra provider call")
					return providerResponse(400, "", nil), nil
				}
				s := tc.steps[count]
				count++
				return providerResponse(s.status, s.content, s.usage), nil
			})
			r := testRuntime(t, transport, intOverride("transient_retries", tc.retries), intOverride("repair_attempts", tc.repairs))
			recorder := &Recorder{}
			out, err := r.workflow(context.Background(), recorder.observer)
			if tc.cause != nil {
				if !errors.Is(err, tc.cause) {
					t.Fatalf("error %v does not match %v", err, tc.cause)
				}
			} else if err != nil {
				t.Fatal(err)
			}
			if out.Accepted != tc.accepted || out.Rejected != tc.rejected || len(out.Warnings) != tc.warnings {
				t.Fatalf("result %+v", out)
			}
			a, rt, rp := recorder.totals()
			if a != tc.attempts || rt != tc.retryCount || rp != tc.repairCount || count != len(tc.steps) {
				t.Fatalf("counts %d/%d/%d; dispatches %d", a, rt, rp, count)
			}
			records := recorder.snapshot()
			serialized, _ := json.Marshal(records)
			if strings.Contains(string(serialized), "synthetic-test-secret") || strings.Contains(string(serialized), "hello world") {
				t.Fatal("sensitive observation")
			}
			for _, o := range records {
				if o.Kind == string(pk.ExecutionAttempt) && o.StatusCode == 503 && o.Usage.Total.Known {
					t.Fatal("invented known usage")
				}
			}
		})
	}
}