Carry one proposal result from generation to the runner
This commit is contained in:
@@ -43,7 +43,9 @@ rendered by ConfigKit, the correction ledger, LLM execution observations, and
|
||||
model interactions. Candidate capture includes proposals later
|
||||
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
|
||||
attempt, validator, and batch coordinates as applicable. Request metadata records
|
||||
these coordinates with the interaction role. Each interaction retains its available
|
||||
these coordinates with the interaction role. Proposal request metadata carries
|
||||
the zero-based response origin (`start_index`) and the ordered `segment_ids`
|
||||
used to interpret response references. Each interaction retains its available
|
||||
prepared request, response, and failure evidence. Ledger mapping preserves empty
|
||||
proposed and applied replacements and distinguishes unavailable candidates from
|
||||
interrupted work.
|
||||
|
||||
@@ -13,8 +13,10 @@ formats; [configuration](../config.md) owns settings; the
|
||||
3. The runner partitions the normalized transcript exactly once before its
|
||||
module loop. Initial section identity, segment membership, and token
|
||||
estimates stay fixed while text evolves.
|
||||
4. Module instances execute in configured order. Sections may propose
|
||||
concurrently, but proposal indices follow section order and response order.
|
||||
4. Module instances execute in configured order. Each section hands its decoded
|
||||
proposals and original local segment references to the runner along with
|
||||
operation facts and any captured exchange. Sections may propose concurrently.
|
||||
The runner assigns indices in section order and response order.
|
||||
5. Deterministic validators precede LLM-backed validators. Every submitted
|
||||
candidate receives one decision per validator on a normal return. Fatal returns
|
||||
may preserve a checked subset of completed decisions as diagnostic evidence.
|
||||
|
||||
@@ -472,6 +472,14 @@ Audit #46.
|
||||
verify indices and original references. Keep model response and reference
|
||||
coverage in the generator's owning tests.
|
||||
|
||||
**Complete.** The generator and runner share one decoded proposal result with
|
||||
raw local references, operation facts, and the optional native exchange. The
|
||||
runner assigns indices in section and response order. A production-module
|
||||
fixture observed the later response first and verified applied indices,
|
||||
operation associations, and raw references. Proposal diagnostics retain
|
||||
the response-local `start_index`. The full offline suite and repeated
|
||||
ordering checks under the race detector passed.
|
||||
|
||||
## Stage 14 — Own each correction repair branch explicitly
|
||||
|
||||
Audit #34, #45, #48, #64 (branch integration).
|
||||
|
||||
@@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 2 || len(generated.Enriched) != 1 {
|
||||
if calls != 2 || len(generated.Proposals) != 1 {
|
||||
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript}
|
||||
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -42,7 +42,6 @@ type Request struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -50,12 +49,8 @@ type Request struct {
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
type Result struct {
|
||||
SegmentReferences []int `json:"-"`
|
||||
Exchange *structuredoutput.Exchange `json:"-"`
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
contracts.ProposalResult
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
@@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if strings.TrimSpace(req.ModuleInstance) == "" {
|
||||
return Result{}, fmt.Errorf("module instance must not be empty")
|
||||
}
|
||||
if req.StartIndex < 0 {
|
||||
return Result{}, fmt.Errorf("start index must be non-negative")
|
||||
}
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("structured LLM client is required")
|
||||
}
|
||||
@@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"start_index": 0, // Position of the first item within this response.
|
||||
"segment_ids": segmentIDs(req.WorkingTranscript),
|
||||
"prompt": completion.Prompt,
|
||||
}
|
||||
@@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
|
||||
if callErr != nil {
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
|
||||
result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts}
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
return result, nil
|
||||
}
|
||||
@@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
|
||||
references := make([]int, 0, len(response.Corrections))
|
||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||
for i, raw := range response.Corrections {
|
||||
for _, raw := range response.Corrections {
|
||||
references = append(references, raw.SegmentReference)
|
||||
// Unknown references remain invalid candidates for the existing domain validators.
|
||||
candidate := proposals.CorrectionProposal{
|
||||
corrections = append(corrections, proposals.CorrectionProposal{
|
||||
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: candidate,
|
||||
ProposalMetadata: proposals.ProposalMetadata{
|
||||
ProposalIndex: req.StartIndex + i,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
},
|
||||
}
|
||||
if req.Section != nil {
|
||||
sectionIndex := req.Section.Index
|
||||
enrichedCandidate.SectionIndex = §ionIndex
|
||||
}
|
||||
enriched = append(enriched, enrichedCandidate)
|
||||
})
|
||||
}
|
||||
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
SegmentReferences: references,
|
||||
Exchange: completion.Exchange,
|
||||
Enriched: enriched,
|
||||
Operations: []outcomes.Operation{operation},
|
||||
Artifacts: artifacts,
|
||||
ProposalResult: contracts.ProposalResult{
|
||||
Proposals: corrections,
|
||||
SegmentReferences: references,
|
||||
Exchange: completion.Exchange,
|
||||
Operations: []outcomes.Operation{operation},
|
||||
},
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request {
|
||||
PromptID: "modules.grammar.proposal",
|
||||
Variables: map[string]string{"SectionJSON": "[]"},
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
|
||||
StartIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
section := contracts.SectionMetadata{Index: 2}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.StartIndex = 10
|
||||
req.Section = §ion
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
|
||||
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
|
||||
t.Fatalf("unexpected proposal lengths: %+v", got)
|
||||
}
|
||||
if got.Corrections[0].TargetSegmentID != 7 {
|
||||
t.Fatalf("local reference not resolved: %+v", got.Corrections)
|
||||
}
|
||||
if got.Enriched[0].ProposalIndex != 10 {
|
||||
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
|
||||
}
|
||||
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
|
||||
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
|
||||
if got.Proposals[0].TargetSegmentID != 7 {
|
||||
t.Fatalf("local reference not resolved: %+v", got.Proposals)
|
||||
}
|
||||
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
|
||||
t.Fatalf("proposal operation identity: %+v", got.Operations)
|
||||
@@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
if err != nil {
|
||||
t.Fatalf("expected invalid correction to survive generation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 1 {
|
||||
if len(result.Proposals) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
if result.Proposals[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
|
||||
}
|
||||
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
|
||||
if err != nil {
|
||||
@@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
|
||||
if err != nil {
|
||||
t.Fatalf("expected recoverable model operation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
if len(result.Proposals) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
|
||||
@@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
t.Fatalf("valid empty response: %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
|
||||
reqA := defaultRequest(t)
|
||||
reqA.LLMClient = clientA
|
||||
reqA.StartIndex = 3
|
||||
first, err := GenerateCandidates(context.Background(), reqA)
|
||||
if err != nil {
|
||||
t.Fatalf("first generation failed: %v", err)
|
||||
}
|
||||
|
||||
reqB := defaultRequest(t)
|
||||
reqB.LLMClient = clientB
|
||||
reqB.StartIndex = 3
|
||||
second, err := GenerateCandidates(context.Background(), reqB)
|
||||
if err != nil {
|
||||
t.Fatalf("second generation failed: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
|
||||
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
section1 := contracts.SectionMetadata{Index: 1}
|
||||
|
||||
req0 := defaultRequest(t)
|
||||
req0.LLMClient = client
|
||||
req0.Section = §ion0
|
||||
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
|
||||
req0.StartIndex = 0
|
||||
part0, err := GenerateCandidates(context.Background(), req0)
|
||||
if err != nil {
|
||||
t.Fatalf("section 0 generation failed: %v", err)
|
||||
}
|
||||
|
||||
req1 := defaultRequest(t)
|
||||
req1.LLMClient = client
|
||||
req1.Section = §ion1
|
||||
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
|
||||
req1.StartIndex = len(part0.Enriched)
|
||||
part1, err := GenerateCandidates(context.Background(), req1)
|
||||
if err != nil {
|
||||
t.Fatalf("section 1 generation failed: %v", err)
|
||||
}
|
||||
|
||||
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 proposals, got %d", len(all))
|
||||
}
|
||||
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
|
||||
t.Fatalf("section references did not reset: %+v", all)
|
||||
}
|
||||
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
|
||||
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
|
||||
}
|
||||
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
|
||||
t.Fatalf("unexpected section metadata: %+v", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
@@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||
t.Fatalf("unmarshal metadata artifact: %v", err)
|
||||
}
|
||||
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
|
||||
t.Fatalf("missing response-local origin: %s", raw)
|
||||
}
|
||||
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
|
||||
@@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected completion failure, got %v", err)
|
||||
}
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
|
||||
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
|
||||
|
||||
@@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
Interactions: req.ProposalRequest.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{Operations: generated.Operations}, err
|
||||
}
|
||||
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
SegmentReferences: generated.SegmentReferences,
|
||||
Exchange: generated.Exchange,
|
||||
Operations: generated.Operations,
|
||||
}, nil
|
||||
return generated.ProposalResult, err
|
||||
}
|
||||
|
||||
@@ -343,10 +343,7 @@ type collectSectionProposalsInput struct {
|
||||
type sectionProposalResult struct {
|
||||
sectionPos int
|
||||
section chunking.Section
|
||||
corrected []proposals.CorrectionProposal
|
||||
references []int
|
||||
exchange *structuredoutput.Exchange
|
||||
operations []outcomes.Operation
|
||||
proposal contracts.ProposalResult
|
||||
started bool
|
||||
err error
|
||||
}
|
||||
@@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
corrected: proposalResult.Proposals,
|
||||
references: proposalResult.SegmentReferences,
|
||||
exchange: proposalResult.Exchange,
|
||||
operations: proposalResult.Operations,
|
||||
proposal: proposalResult,
|
||||
started: true,
|
||||
err: err,
|
||||
}
|
||||
@@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if result.started {
|
||||
unit.State = outcomes.WorkInterrupted
|
||||
}
|
||||
unit.Operations = append(unit.Operations, result.operations...)
|
||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||
if result.err != nil {
|
||||
setErr(result.err)
|
||||
continue
|
||||
@@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
delete(pending, nextSectionToProcess)
|
||||
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
|
||||
for i, corrected := range sectionResult.corrected {
|
||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
||||
for i, corrected := range sectionResult.proposal.Proposals {
|
||||
sectionIndex := sectionMeta.Index
|
||||
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: corrected,
|
||||
@@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
Exchange: sectionResult.proposal.Exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: input.Interactions,
|
||||
@@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
|
||||
promptID = provider.RepairPromptID()
|
||||
}
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
|
||||
}
|
||||
setErr(err)
|
||||
validationResults <- sectionValidationResult{
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
@@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 3
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: "alpha one"},
|
||||
{ID: 2, Text: "bravo two"},
|
||||
{ID: 3, Text: "charlie three"},
|
||||
}}
|
||||
|
||||
started := make(chan int, len(transcript.Segments))
|
||||
releaseByID := map[int]chan struct{}{
|
||||
1: make(chan struct{}),
|
||||
2: make(chan struct{}),
|
||||
3: make(chan struct{}),
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
if len(req.WorkingTranscript.Segments) != 1 {
|
||||
t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments))
|
||||
}
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
started <- seg.ID
|
||||
<-releaseByID[seg.ID]
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
|
||||
resultCh := make(chan struct {
|
||||
out RunOutput
|
||||
err error
|
||||
}, 1)
|
||||
go func() {
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: transcript,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{
|
||||
{ModuleKey: "m", InstanceName: "m"},
|
||||
},
|
||||
})
|
||||
resultCh <- struct {
|
||||
out RunOutput
|
||||
err error
|
||||
}{out: out, err: err}
|
||||
}()
|
||||
|
||||
waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}})
|
||||
close(releaseByID[3])
|
||||
close(releaseByID[2])
|
||||
close(releaseByID[1])
|
||||
|
||||
result := <-resultCh
|
||||
out, err := result.out, result.err
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 {
|
||||
t.Fatalf("expected one module result, got %+v", out.ModuleResults)
|
||||
}
|
||||
if got := len(out.ModuleResults[0].AppliedChanges); got != 3 {
|
||||
t.Fatalf("expected three applied changes, got %d", got)
|
||||
}
|
||||
for i, change := range out.ModuleResults[0].AppliedChanges {
|
||||
wantID := i + 1
|
||||
if change.TargetSegmentID != wantID {
|
||||
t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
|
||||
}
|
||||
}
|
||||
wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
|
||||
for i, seg := range out.FinalTranscript.Segments {
|
||||
if seg.Text != wantFinal[i] {
|
||||
t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
|
||||
t.Helper()
|
||||
deadline := time.After(350 * time.Millisecond)
|
||||
@@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type proposalGenerationModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error)
|
||||
|
||||
func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
return f(ctx, req, out)
|
||||
}
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
PromptID: "modules.grammar.proposal",
|
||||
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
|
||||
LLMClient: req.LLMClient,
|
||||
Interactions: req.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
responses []proposal_generation.StructuredCorrectionSet
|
||||
}
|
||||
|
||||
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
client := &fakeProposalStructuredClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
cfg.Chunking.MaxSectionTokens = 3
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 71, Text: "alpha one"},
|
||||
{ID: 903, Text: "bravo two"},
|
||||
}}
|
||||
responses := []proposal_generation.StructuredCorrectionSet{
|
||||
{Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
}},
|
||||
{Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
}},
|
||||
}
|
||||
completedLaterSection := make(chan struct{})
|
||||
client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if req.Context.SectionIndex == nil {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("missing section index")
|
||||
}
|
||||
index := *req.Context.SectionIndex
|
||||
if index < 0 || index >= len(responses) {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index")
|
||||
}
|
||||
if index == 0 {
|
||||
select {
|
||||
case <-completedLaterSection:
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
*out.(*proposal_generation.StructuredCorrectionSet) = responses[index]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
})
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
|
||||
}})
|
||||
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
|
||||
Interactions: llm.NewDiagnosticsWriter(diagDir),
|
||||
dir := t.TempDir()
|
||||
writer := llm.NewDiagnosticsWriter(dir)
|
||||
interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := r.Run(ctx, RunInput{
|
||||
Config: &cfg, Transcript: transcript, Interactions: interactions,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected run error: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
want := []struct {
|
||||
target int
|
||||
original string
|
||||
}{{71, "one"}, {71, "alpha"}, {903, "two"}}
|
||||
changes := out.ModuleResults[0].AppliedChanges
|
||||
if len(changes) != len(want) {
|
||||
t.Fatalf("applied changes: %+v", changes)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
|
||||
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
|
||||
for i, change := range changes {
|
||||
if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original {
|
||||
t.Fatalf("change %d lost response order or segment identity: %+v", i, change)
|
||||
}
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob diagnostics: %v", globErr)
|
||||
wantIndices := [][]int{{0, 1}, {2}}
|
||||
if len(out.WorkUnits) != len(wantIndices) {
|
||||
t.Fatalf("work units: %+v", out.WorkUnits)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
|
||||
for i, unit := range out.WorkUnits {
|
||||
if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) {
|
||||
t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations)
|
||||
}
|
||||
}
|
||||
|
||||
paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json"))
|
||||
if err != nil || len(paths) != len(responses) {
|
||||
t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err)
|
||||
}
|
||||
for i, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var evidence struct {
|
||||
Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(evidence.Decoded, responses[i]) {
|
||||
t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type proposalOrderSink struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
laterResponse chan struct{}
|
||||
}
|
||||
|
||||
func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
artifacts, err := s.writer.WriteInteraction(interaction)
|
||||
if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 {
|
||||
close(s.laterResponse)
|
||||
}
|
||||
return artifacts, err
|
||||
}
|
||||
|
||||
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user