Files
audita/internal/framework/proposal_generation/generate_test.go
Eric Rakestraw da5b8913d1
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Migrate Audita to the v4 module path
2026-11-17 19:57:22 +00:00

349 lines
13 KiB
Go

package proposal_generation
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/structuredoutput"
)
type fakeStructuredClient struct {
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
calls []contracts.StructuredCompletionRequest
}
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return f.completion, f.err
}
target, ok := out.(*StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
*target = f.responses[0]
f.responses = f.responses[1:]
var content strings.Builder
for _, value := range req.Variables {
content.WriteString(value)
}
response := f.completion
response.Prepared = &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}
return response, nil
}
type captureDiagnosticsWriter struct {
err error
lastStage string
lastRequestMetadata any
lastRequestPayload any
}
func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
w.lastStage = interaction.Name
w.lastRequestMetadata = interaction.RequestMetadata
w.lastRequestPayload = interaction.RequestPayload
return structuredoutput.InteractionArtifacts{}, w.err
}
func defaultRequest(t *testing.T) Request {
t.Helper()
return Request{
ModuleKey: "test_module",
ModuleInstance: "test_module",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
}
}
func TestGenerateCandidatesSuccess(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
}
section := contracts.SectionMetadata{Index: 2}
req := defaultRequest(t)
req.LLMClient = client
req.Section = &section
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Proposals[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Proposals)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
}
}
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
req := defaultRequest(t)
req.LLMClient = client
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(client.calls) != 1 {
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
}
call := client.calls[0]
if call.Schema != responseschema.CorrectionSetKey || call.PromptID != req.PromptID || call.Context.Role != "proposal" {
t.Fatalf("unexpected call contract: %+v", call)
}
}
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
diag := &captureDiagnosticsWriter{}
req := defaultRequest(t)
req.LLMClient = client
req.Interactions = diag
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
metadata, ok := diag.lastRequestMetadata.(map[string]any)
if !ok {
t.Fatalf("expected request metadata map, got %T", diag.lastRequestMetadata)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata map, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
}
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 9, OriginalText: "x", CorrectedText: ""},
},
},
},
}
req := defaultRequest(t)
req.LLMClient = client
req.Interactions = llm.NewDiagnosticsWriter(t.TempDir())
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Proposals) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Proposals[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if len(result.SegmentReferences) != 1 || result.SegmentReferences[0] != 9 ||
len(evidence.Decoded.Corrections) != 1 || evidence.Decoded.Corrections[0].SegmentReference != 9 {
t.Fatalf("lost model reference: result=%v artifact=%s", result.SegmentReferences, raw)
}
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
}
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Proposals) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
t.Fatalf("expected one failed operation, got %+v", result.Operations)
}
}
func TestGenerateCandidatesUsesInternalReasonForUnspecifiedRecoverableFailure(t *testing.T) {
cause := errors.New("underlying call failure")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{err: &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Cause: cause}}
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Operations) != 1 {
t.Fatalf("recoverable proposal returned operations=%+v error=%v", result.Operations, err)
}
operation := result.Operations[0]
if operation.FailureClass != outcomes.FailureRecoverable || operation.FailureReason != outcomes.ReasonInternalFailure || !errors.Is(operation.Cause, cause) {
t.Fatalf("operation lost typed failure facts: %+v", operation)
}
}
func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonSetupFailure || !result.Operations[0].Counts.Available {
t.Fatalf("fatal setup lost call facts: %+v, %v", result, err)
}
}
func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}},
},
}
req := defaultRequest(t)
req.LLMClient = client
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
req.Variables = map[string]string{"SectionJSON": "section text"}
req.Section = &contracts.SectionMetadata{Index: 3}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" {
t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts)
}
for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr)
}
if !json.Valid(raw) {
t.Fatalf("invalid JSON artifact %q: %s", path, raw)
}
}
raw, readErr := os.ReadFile(got.Artifacts.RequestMetadataPath)
if readErr != nil {
t.Fatalf("read metadata artifact %q: %v", got.Artifacts.RequestMetadataPath, readErr)
}
var metadata map[string]any
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
t.Fatalf("missing response-local origin: %s", raw)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
}
var coordinates struct {
Context structuredoutput.DiagnosticContext `json:"context"`
}
if err := json.Unmarshal(raw, &coordinates); err != nil {
t.Fatal(err)
}
if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 {
t.Fatalf("proposal metadata: %s", raw)
}
}
func TestGenerateCandidatesClientError(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("boom")}
req := defaultRequest(t)
req.LLMClient = client
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
got, err := GenerateCandidates(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected completion failure, got %v", err)
}
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(matches) == 0 {
t.Fatalf("expected error diagnostics artifact under %s", dir)
}
}
func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
sink := &captureDiagnosticsWriter{err: cause}
req.Interactions = sink
result, err := GenerateCandidates(context.Background(), req)
if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
}
}