Unify optional interaction capture and warnings
This commit is contained in:
@@ -7,12 +7,10 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
@@ -20,19 +18,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
// InteractionArtifacts contains written diagnostics paths.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
SegmentReference int `json:"id"`
|
||||
@@ -48,23 +33,20 @@ type StructuredCorrectionSet struct {
|
||||
|
||||
// Request captures reusable proposal-generation inputs for future modules.
|
||||
type Request struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
}
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
@@ -74,7 +56,7 @@ type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
@@ -105,13 +87,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
response StructuredCorrectionSet
|
||||
completion contracts.StructuredCompletionResponse
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
artifacts structuredoutput.InteractionArtifacts
|
||||
)
|
||||
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
|
||||
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
CaptureExchange: req.CaptureExchange,
|
||||
PromptID: req.PromptID,
|
||||
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
|
||||
Context: coordinates,
|
||||
}, &response)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -126,16 +109,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
Counts: completion.Execution,
|
||||
}
|
||||
|
||||
var writer InteractionDiagnosticsWriter
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
|
||||
}
|
||||
}
|
||||
|
||||
if writer != nil {
|
||||
if req.Interactions != nil {
|
||||
requestMetadata := map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID,
|
||||
"module_key": req.ModuleKey,
|
||||
@@ -147,17 +121,11 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
"prompt": completion.Prompt,
|
||||
}
|
||||
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
|
||||
var diagnosticErr error
|
||||
artifacts, diagnosticErr = writer.WriteInteraction(
|
||||
stage,
|
||||
requestMetadata,
|
||||
completion.Prepared,
|
||||
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
|
||||
errPayload(callErr),
|
||||
)
|
||||
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
|
||||
req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr))
|
||||
}
|
||||
artifacts, _ = req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: stage, Context: coordinates, RequestMetadata: requestMetadata, RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
|
||||
ErrorPayload: errPayload(callErr),
|
||||
})
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
@@ -234,23 +202,6 @@ func sectionIndexPtr(section *contracts.SectionMetadata) *int {
|
||||
return &index
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
}
|
||||
return InteractionArtifacts{
|
||||
RequestMetadataPath: artifacts.RequestMetadataPath,
|
||||
RequestPayloadPath: artifacts.RequestPayloadPath,
|
||||
ResponsePayloadPath: artifacts.ResponsePayloadPath,
|
||||
ErrorPayloadPath: artifacts.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// segmentIDs records the request-local reference mapping in debug metadata.
|
||||
func segmentIDs(transcript *schema.Transcript) []int {
|
||||
if transcript == nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
@@ -56,13 +57,11 @@ type captureDiagnosticsWriter struct {
|
||||
lastRequestPayload any
|
||||
}
|
||||
|
||||
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
w.lastStage = stage
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastRequestPayload = requestPayload
|
||||
_ = responsePayload
|
||||
_ = errorPayload
|
||||
return InteractionArtifacts{}, w.err
|
||||
func (w *captureDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
w.lastStage = interaction.Name
|
||||
w.lastRequestMetadata = interaction.RequestMetadata
|
||||
w.lastRequestPayload = interaction.RequestPayload
|
||||
return structuredoutput.InteractionArtifacts{}, w.err
|
||||
}
|
||||
|
||||
func defaultRequest(t *testing.T) Request {
|
||||
@@ -146,7 +145,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
diag := &captureDiagnosticsWriter{}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = diag
|
||||
req.Interactions = diag
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -316,8 +315,10 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
dir := t.TempDir()
|
||||
req.Interactions = llm.NewDiagnosticsWriter(dir)
|
||||
req.Variables = map[string]string{"SectionJSON": "section text"}
|
||||
req.Section = &contracts.SectionMetadata{Index: 3}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -353,13 +354,23 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
var coordinates struct {
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &coordinates); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c := coordinates.Context; c.Role != "proposal" || c.ModuleInstance != req.ModuleInstance || c.SectionIndex == nil || *c.SectionIndex != 3 {
|
||||
t.Fatalf("proposal metadata: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("boom")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
dir := t.TempDir()
|
||||
req.Interactions = llm.NewDiagnosticsWriter(dir)
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
@@ -368,12 +379,12 @@ func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
|
||||
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
|
||||
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob error: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
|
||||
t.Fatalf("expected error diagnostics artifact under %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,11 +392,10 @@ func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
|
||||
req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
sink := &captureDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
|
||||
if err != nil || sink.lastStage == "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,19 +54,18 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
|
||||
generated, err := GenerateCandidates(ctx, Request{
|
||||
StageIndex: req.ProposalRequest.StageIndex, StageID: req.ProposalRequest.StageID,
|
||||
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
|
||||
ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError,
|
||||
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
Interactions: req.ProposalRequest.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{Operations: generated.Operations}, err
|
||||
|
||||
Reference in New Issue
Block a user