Adopt PromptKit v0.23.0 conversations and execution accounting
This commit is contained in:
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
@@ -117,8 +118,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
return findings
|
||||
}
|
||||
findings := safeFindings(rejected)
|
||||
replay := &structuredoutput.Replay{Exchange: input.Exchange}
|
||||
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
|
||||
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
|
||||
if !ok {
|
||||
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
|
||||
}
|
||||
buildRequest := func() (*structuredoutput.Request, error) {
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: position + 1,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
@@ -134,12 +138,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
||||
}
|
||||
exhausted, coordinationErr := coordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response == nil {
|
||||
return buildRequest()
|
||||
}
|
||||
completion, callErr := *turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
|
||||
if callErr == nil {
|
||||
_ = json.Unmarshal(completion.Content, &response)
|
||||
}
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
|
||||
@@ -183,26 +198,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
|
||||
if structuredoutput.IsRecoverable(callErr) {
|
||||
branch.Outcome = "unavailable"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
branches = append(branches, branch)
|
||||
return finish(callErr)
|
||||
return nil, callErr
|
||||
}
|
||||
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
operation.FailureReason = outcomes.ReasonInvalidOutput
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "unavailable"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
|
||||
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
|
||||
if decoded.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
revision := decoded.Correction
|
||||
localRef = revision.SegmentReference
|
||||
@@ -225,7 +237,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
for i := range validated.operations {
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attemptNumber
|
||||
validated.operations[i].Reference.ProposalIndex = &proposalIndex
|
||||
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
||||
}
|
||||
@@ -236,32 +248,35 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
finishAttempt(validationErr, ¤t, validated)
|
||||
initial.operations = append(initial.operations, validated.operations...)
|
||||
if validationErr != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(validationErr)
|
||||
return nil, validationErr
|
||||
}
|
||||
if approved {
|
||||
branch.Outcome = "approved"
|
||||
initial.approved = append(initial.approved, current)
|
||||
superseded[original.ProposalIndex] = true
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
if attempt+1 == input.Spec.Settings.LLM.CorrectionRepairAttempts {
|
||||
branch.Outcome = "exhausted"
|
||||
break
|
||||
}
|
||||
findings = safeFindings(validated.rejected)
|
||||
continue
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
for _, d := range validated.rejected {
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
branch.Outcome = "rejected"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
break
|
||||
return nil, nil
|
||||
})
|
||||
if exhausted {
|
||||
branch.Outcome = "exhausted"
|
||||
}
|
||||
if coordinationErr != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(coordinationErr)
|
||||
}
|
||||
|
||||
branches = append(branches, branch)
|
||||
}
|
||||
return finish(nil)
|
||||
|
||||
@@ -4,10 +4,17 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
@@ -20,6 +27,7 @@ import (
|
||||
)
|
||||
|
||||
type repairClient struct {
|
||||
t *testing.T
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
@@ -27,11 +35,7 @@ type repairClient struct {
|
||||
}
|
||||
|
||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
if req.Replay != nil {
|
||||
copyReplay := *req.Replay
|
||||
copyReplay.Turns = append([]structuredoutput.Message(nil), req.Replay.Turns...)
|
||||
req.Replay = ©Replay
|
||||
}
|
||||
|
||||
c.calls = append(c.calls, req)
|
||||
if c.err != nil {
|
||||
return structuredoutput.Response{}, c.err
|
||||
@@ -40,18 +44,18 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
||||
if err := json.Unmarshal([]byte(value), out); err != nil {
|
||||
return structuredoutput.Response{}, err
|
||||
}
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}
|
||||
|
||||
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -85,7 +89,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
@@ -93,7 +97,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -125,7 +129,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{
|
||||
client := &repairClient{t: t, outputs: []string{
|
||||
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
|
||||
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
|
||||
}}
|
||||
@@ -138,7 +142,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -175,9 +179,9 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: tc.outputs, err: tc.err}
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -189,8 +193,8 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
|
||||
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
|
||||
}
|
||||
if tc.calls == 2 && len(client.calls[1].Replay.Turns) != 2 {
|
||||
t.Fatalf("history: %+v", client.calls[1].Replay.Turns)
|
||||
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
|
||||
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -202,11 +206,11 @@ func (m repairModule) RepairPromptID() string { return "repair" }
|
||||
|
||||
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
|
||||
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}, nil
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
@@ -246,11 +250,11 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -271,9 +275,9 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}
|
||||
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -285,12 +289,12 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -318,14 +322,14 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
path := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warnings := 0
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -337,7 +341,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
client := &repairClient{}
|
||||
client := &repairClient{t: t}
|
||||
for _, rejected := range [][]ValidatorRejectedChange{
|
||||
{{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}},
|
||||
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
|
||||
@@ -352,10 +356,10 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
validation := &repairClient{secrets: []string{"validation-secret"}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil {
|
||||
@@ -393,3 +397,66 @@ func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
type repairTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f repairTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
// Exercise the real adapter and upstream coordinator; only model responses are scripted.
|
||||
func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
|
||||
t := c.t
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var selected *structuredoutput.Request
|
||||
initial := true
|
||||
runtime, err := llm.NewRuntime(ctx, loaded, &http.Client{Transport: repairTransport(func(*http.Request) (*http.Response, error) {
|
||||
content := `{"corrections":[]}`
|
||||
if initial {
|
||||
initial = false
|
||||
} else {
|
||||
c.calls = append(c.calls, *selected)
|
||||
content = c.outputs[len(c.calls)-1]
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}})
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body)))}, nil
|
||||
})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("repair", "fixture", config.LLM{OperationTimeout: time.Second}, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output map[string]any
|
||||
base, err := client.CompleteStructured(ctx, structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, CaptureExchange: true, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}, &output)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return client.Coordinate(ctx, base.Exchange, budget, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
req, err := next(ctx, turn)
|
||||
if req != nil && err == nil {
|
||||
selected = req
|
||||
if c.err != nil {
|
||||
c.calls = append(c.calls, *req)
|
||||
_, err = next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: c.err})
|
||||
return nil, err
|
||||
}
|
||||
// Test prompt IDs vary; all use the same supplied-data fixture definition.
|
||||
copy := *req
|
||||
copy.PromptID = "repair"
|
||||
return ©, nil
|
||||
}
|
||||
return req, err
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user