Unify optional interaction capture and warnings

This commit is contained in:
2026-11-16 03:43:29 +00:00
parent ffe0e3491e
commit cea7e44232
22 changed files with 450 additions and 367 deletions

View File

@@ -7,12 +7,10 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
@@ -20,19 +18,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
type InteractionDiagnosticsWriter interface {
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
}
// InteractionArtifacts contains written diagnostics paths.
type InteractionArtifacts struct {
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
}
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
SegmentReference int `json:"id"`
@@ -48,23 +33,20 @@ type StructuredCorrectionSet struct {
// Request captures reusable proposal-generation inputs for future modules.
type Request struct {
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
DiagnosticsDir string
DiagnosticsWriter InteractionDiagnosticsWriter
StageIndex *int `json:"stage_index,omitempty"`
StageID *string `json:"stage_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
}
// Result contains generated candidate proposals and optional diagnostics paths.
@@ -74,7 +56,7 @@ type Result struct {
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
}
// GenerateCandidates executes one structured LLM call and deterministically maps
@@ -105,13 +87,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
response StructuredCorrectionSet
completion contracts.StructuredCompletionResponse
callErr error
artifacts InteractionArtifacts
artifacts structuredoutput.InteractionArtifacts
)
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
CaptureExchange: req.CaptureExchange,
PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
Context: coordinates,
}, &response)
sectionIndex := 0
if req.Section != nil {
@@ -126,16 +109,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
Counts: completion.Execution,
}
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
}
}
if writer != nil {
if req.Interactions != nil {
requestMetadata := map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID,
"module_key": req.ModuleKey,
@@ -147,17 +121,11 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction(
stage,
requestMetadata,
completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(callErr),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr))
}
artifacts, _ = req.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: stage, Context: coordinates, RequestMetadata: requestMetadata, RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
ErrorPayload: errPayload(callErr),
})
}
if callErr != nil {
@@ -234,23 +202,6 @@ func sectionIndexPtr(section *contracts.SectionMetadata) *int {
return &index
}
type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return InteractionArtifacts{}, err
}
return InteractionArtifacts{
RequestMetadataPath: artifacts.RequestMetadataPath,
RequestPayloadPath: artifacts.RequestPayloadPath,
ResponsePayloadPath: artifacts.ResponsePayloadPath,
ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil
}
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
@@ -56,13 +57,11 @@ type captureDiagnosticsWriter struct {
lastRequestPayload any
}
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
w.lastStage = stage
w.lastRequestMetadata = requestMetadata
w.lastRequestPayload = requestPayload
_ = responsePayload
_ = errorPayload
return InteractionArtifacts{}, w.err
func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
w.lastStage = interaction.Name
w.lastRequestMetadata = interaction.RequestMetadata
w.lastRequestPayload = interaction.RequestPayload
return structuredoutput.InteractionArtifacts{}, w.err
}
func defaultRequest(t *testing.T) Request {
@@ -146,7 +145,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
diag := &captureDiagnosticsWriter{}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsWriter = diag
req.Interactions = diag
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -316,8 +315,10 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
req.Variables = map[string]string{"SectionJSON": "section text"}
req.Section = &contracts.SectionMetadata{Index: 3}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -353,13 +354,23 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
}
var coordinates struct {
Context structuredoutput.DiagnosticContext `json:"context"`
}
if err := json.Unmarshal(raw, &coordinates); err != nil {
t.Fatal(err)
}
if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 {
t.Fatalf("proposal metadata: %s", raw)
}
}
func TestGenerateCandidatesClientError(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("boom")}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
dir := t.TempDir()
req.Interactions = llm.NewDiagnosticsWriter(dir)
got, err := GenerateCandidates(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") {
@@ -368,12 +379,12 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(matches) == 0 {
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
t.Fatalf("expected error diagnostics artifact under %s", dir)
}
}
@@ -381,11 +392,10 @@ func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
sink := &captureDiagnosticsWriter{err: cause}
req.Interactions = sink
result, err := GenerateCandidates(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
}
}

View File

@@ -54,19 +54,18 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
generated, err := GenerateCandidates(ctx, Request{
StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
Interactions: req.ProposalRequest.Interactions,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err