Remove retired runtime wrappers and persistent cache machinery
This commit is contained in:
@@ -1,84 +0,0 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// DebugStructuredCompletionRequest is the content-safe representation of a
|
||||
// structured completion request for ordinary diagnostics and summaries.
|
||||
// Detailed prompt material remains available only through the explicitly
|
||||
// requested LLM debug trace.
|
||||
type DebugStructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
InputCount int `json:"input_count"`
|
||||
VariableCount int `json:"variable_count"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Correction *DebugSemanticCorrection `json:"correction,omitempty"`
|
||||
}
|
||||
|
||||
// DebugSemanticCorrection records only safe correction metadata. It never
|
||||
// exposes the assistant response or user guidance text.
|
||||
type DebugSemanticCorrection struct {
|
||||
AssistantResponseBytes int `json:"assistant_response_bytes"`
|
||||
AssistantResponseDigest string `json:"assistant_response_digest"`
|
||||
UserGuidanceBytes int `json:"user_guidance_bytes"`
|
||||
UserGuidanceDigest string `json:"user_guidance_digest"`
|
||||
}
|
||||
|
||||
// DebugSummary returns a content-safe representation suitable for ordinary
|
||||
// diagnostics. It does not validate or retain correction content.
|
||||
func (request StructuredCompletionRequest) DebugSummary() DebugStructuredCompletionRequest {
|
||||
summary := DebugStructuredCompletionRequest{
|
||||
StageName: request.StageName,
|
||||
PromptID: request.PromptID,
|
||||
PromptVersion: request.PromptVersion,
|
||||
ProfileID: request.ProfileID,
|
||||
SessionID: request.SessionID,
|
||||
InputCount: len(request.Inputs),
|
||||
VariableCount: len(request.Vars),
|
||||
}
|
||||
if request.StructuredOutputRepairAttempts != nil {
|
||||
attempts := *request.StructuredOutputRepairAttempts
|
||||
summary.StructuredOutputRepairAttempts = &attempts
|
||||
}
|
||||
if request.Correction != nil {
|
||||
summary.Correction = request.Correction.DebugSummary()
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// DebugSummary returns content-safe correction metadata suitable for ordinary
|
||||
// diagnostics.
|
||||
func (correction *SemanticCorrection) DebugSummary() *DebugSemanticCorrection {
|
||||
if correction == nil {
|
||||
return nil
|
||||
}
|
||||
return &DebugSemanticCorrection{
|
||||
AssistantResponseBytes: len(correction.AssistantResponse),
|
||||
AssistantResponseDigest: debugContentDigest(correction.AssistantResponse),
|
||||
UserGuidanceBytes: len(correction.UserGuidance),
|
||||
UserGuidanceDigest: debugContentDigest([]byte(correction.UserGuidance)),
|
||||
}
|
||||
}
|
||||
|
||||
// String prevents ordinary request formatting from exposing correction
|
||||
// content. Use the explicitly requested debug trace for complete messages.
|
||||
func (request StructuredCompletionRequest) String() string {
|
||||
return fmt.Sprintf("%+v", request.DebugSummary())
|
||||
}
|
||||
|
||||
// GoString gives %#v formatting the same content-safe behavior as String.
|
||||
func (request StructuredCompletionRequest) GoString() string {
|
||||
return request.String()
|
||||
}
|
||||
|
||||
func debugContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -7,122 +7,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Correction *SemanticCorrection `json:"-"`
|
||||
}
|
||||
|
||||
type StructuredCompletionResponse struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts,omitempty"`
|
||||
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
||||
Execution *LLMExecutionSummary `json:"execution,omitempty"`
|
||||
}
|
||||
|
||||
type LLMObservedTokenCount struct {
|
||||
Value int `json:"value"`
|
||||
Known bool `json:"known"`
|
||||
Complete bool `json:"complete"`
|
||||
}
|
||||
|
||||
type LLMObservedTokenUsage struct {
|
||||
PromptTokens LLMObservedTokenCount `json:"prompt_tokens"`
|
||||
CompletionTokens LLMObservedTokenCount `json:"completion_tokens"`
|
||||
TotalTokens LLMObservedTokenCount `json:"total_tokens"`
|
||||
CachedTokens LLMObservedTokenCount `json:"cached_tokens"`
|
||||
CacheWriteTokens LLMObservedTokenCount `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
type LLMExecutionSummary struct {
|
||||
Attempts uint64 `json:"attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
RepairCalls uint64 `json:"repair_calls"`
|
||||
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
||||
GenerationMS int64 `json:"generation_ms"`
|
||||
RetryDelayMS int64 `json:"retry_delay_ms"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Outcome string `json:"outcome"`
|
||||
FailureKind string `json:"failure_kind"`
|
||||
StatusCode int `json:"status_code"`
|
||||
RetriesExhausted bool `json:"retries_exhausted"`
|
||||
Usage LLMObservedTokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
type StructuredLLMClient interface {
|
||||
CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error)
|
||||
}
|
||||
|
||||
type LLMDebugMaterial struct {
|
||||
Prompt *LLMDebugPrompt `json:"prompt,omitempty"`
|
||||
Response *LLMDebugResponse `json:"response,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugPrompt struct {
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
Messages []LLMDebugMessage `json:"messages,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
OutputContract map[string]any `json:"output_contract,omitempty"`
|
||||
StructuredOutput map[string]any `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl map[string]any `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Validation map[string]any `json:"validation,omitempty"`
|
||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||
ProviderError *LLMDebugProviderError `json:"provider_error,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugProviderError struct {
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type LLMInputMaterial struct {
|
||||
Name string `json:"name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
@@ -162,12 +46,11 @@ func (set LLMInputSet) Clone() LLMInputSet {
|
||||
}
|
||||
|
||||
type ParseRequest struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw []byte `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw []byte `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type InputAdapter interface {
|
||||
|
||||
@@ -14,7 +14,6 @@ var _ Chunker = fakeChunker{}
|
||||
var _ Extractor[fakeArtifact] = fakeExtractor{}
|
||||
var _ Merger[fakeArtifact] = fakeMerger{}
|
||||
var _ Normalizer[fakeArtifact] = fakeNormalizer{}
|
||||
var _ StructuredLLMClient = fakeLLMClient{}
|
||||
var _ OutputEncoder = fakeOutputEncoder{}
|
||||
|
||||
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
|
||||
@@ -314,14 +313,6 @@ func (normalizer fakeNormalizer) Normalize(ctx context.Context, req TypedNormali
|
||||
return TypedNormalizeResult[fakeArtifact]{Value: req.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
type fakeLLMClient struct{}
|
||||
|
||||
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
||||
return StructuredCompletionResponse{
|
||||
Content: json.RawMessage(`{"value":"example"}`),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fakeOutputEncoder struct {
|
||||
key string
|
||||
}
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
// CorrectionProtocol identifies how a producer can represent the model output
|
||||
// that directly controlled a candidate.
|
||||
type CorrectionProtocol string
|
||||
|
||||
const (
|
||||
CorrectionProtocolSingleResponseV1 CorrectionProtocol = "single_response_v1"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxValidationReasonCodeBytes = 128
|
||||
MaxValidationCorrectionGuidanceBytes = 4 * 1024
|
||||
MaxAssistantResponseBytes = 1 << 20
|
||||
MaxCorrectionGuidanceBytes = 64 * 1024
|
||||
MaxCorrectionContentBytes = MaxAssistantResponseBytes + MaxCorrectionGuidanceBytes
|
||||
)
|
||||
|
||||
func (protocol CorrectionProtocol) Validate() error {
|
||||
if protocol == CorrectionProtocolSingleResponseV1 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unsupported correction protocol %q", protocol)
|
||||
}
|
||||
|
||||
// SemanticCorrection carries the latest model response and application-owned
|
||||
// guidance for one fresh corrected request. Its content is sensitive and is
|
||||
// deliberately excluded from ordinary JSON serialization.
|
||||
type SemanticCorrection struct {
|
||||
AssistantResponse []byte `json:"-"`
|
||||
UserGuidance string `json:"-"`
|
||||
}
|
||||
|
||||
func NewSemanticCorrection(assistantResponse []byte, userGuidance string) (*SemanticCorrection, error) {
|
||||
correction := &SemanticCorrection{
|
||||
AssistantResponse: append([]byte(nil), assistantResponse...),
|
||||
UserGuidance: userGuidance,
|
||||
}
|
||||
if err := correction.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return correction, nil
|
||||
}
|
||||
|
||||
func CloneSemanticCorrection(correction *SemanticCorrection) (*SemanticCorrection, error) {
|
||||
if correction == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return NewSemanticCorrection(correction.AssistantResponse, correction.UserGuidance)
|
||||
}
|
||||
|
||||
// CloneStructuredCompletionRequest returns a request whose mutable values are
|
||||
// owned by the caller. It is suitable for clients that retain requests after
|
||||
// CompleteStructured returns.
|
||||
func CloneStructuredCompletionRequest(request StructuredCompletionRequest) (StructuredCompletionRequest, error) {
|
||||
correction, err := CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return StructuredCompletionRequest{}, fmt.Errorf("clone correction: %w", err)
|
||||
}
|
||||
vars, err := source.CloneMetadata(request.Vars)
|
||||
if err != nil {
|
||||
return StructuredCompletionRequest{}, fmt.Errorf("clone variables: %w", err)
|
||||
}
|
||||
request.Inputs = request.Inputs.Clone()
|
||||
request.Vars = vars
|
||||
request.Correction = correction
|
||||
if request.StructuredOutputRepairAttempts != nil {
|
||||
attempts := *request.StructuredOutputRepairAttempts
|
||||
request.StructuredOutputRepairAttempts = &attempts
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func (correction SemanticCorrection) Validate() error {
|
||||
if err := validateAssistantResponse(correction.AssistantResponse); err != nil {
|
||||
return fmt.Errorf("semantic correction assistant response: %w", err)
|
||||
}
|
||||
if err := validateBoundedText(correction.UserGuidance, MaxCorrectionGuidanceBytes, "semantic correction user guidance", false); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(correction.AssistantResponse)+len(correction.UserGuidance) > MaxCorrectionContentBytes {
|
||||
return errors.New("semantic correction content exceeds maximum length")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModelCandidate preserves the exact single model response that directly
|
||||
// controlled a producer result. It is attempt-local and never durable data.
|
||||
type ModelCandidate struct {
|
||||
Response []byte `json:"-"`
|
||||
Protocol CorrectionProtocol `json:"-"`
|
||||
}
|
||||
|
||||
func NewModelCandidate(response []byte, protocol CorrectionProtocol) (*ModelCandidate, error) {
|
||||
candidate := &ModelCandidate{Response: append([]byte(nil), response...), Protocol: protocol}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func CloneModelCandidate(candidate *ModelCandidate) (*ModelCandidate, error) {
|
||||
if candidate == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return NewModelCandidate(candidate.Response, candidate.Protocol)
|
||||
}
|
||||
|
||||
func (candidate ModelCandidate) Validate() error {
|
||||
if err := candidate.Protocol.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAssistantResponse(candidate.Response); err != nil {
|
||||
return fmt.Errorf("model candidate response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateValidationResult(result ValidationResult) error {
|
||||
if err := ValidateProducerDiagnostics(result.Diagnostics); err != nil {
|
||||
return fmt.Errorf("validation diagnostics: %w", err)
|
||||
}
|
||||
if !result.Approved && result.ReasonCode == "" {
|
||||
return errors.New("validation rejection reason code must not be empty")
|
||||
}
|
||||
if result.ReasonCode != "" {
|
||||
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !result.Approved && result.CorrectionGuidance == "" {
|
||||
return errors.New("validation rejection correction guidance must not be empty")
|
||||
}
|
||||
if result.CorrectionGuidance != "" {
|
||||
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAssistantResponse(response []byte) error {
|
||||
if len(response) > MaxAssistantResponseBytes {
|
||||
return errors.New("exceeds maximum length")
|
||||
}
|
||||
if !utf8.Valid(response) {
|
||||
return errors.New("must be valid UTF-8")
|
||||
}
|
||||
if len(strings.TrimSpace(string(response))) == 0 {
|
||||
return errors.New("must not be blank")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBoundedText(value string, maximum int, name string, optional bool) error {
|
||||
if value == "" && optional {
|
||||
return nil
|
||||
}
|
||||
if !utf8.ValidString(value) {
|
||||
return fmt.Errorf("%s must be valid UTF-8", name)
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("%s must not be blank", name)
|
||||
}
|
||||
if len(value) > maximum {
|
||||
return fmt.Errorf("%s exceeds maximum length", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSemanticCorrectionOwnsValidatedContent(t *testing.T) {
|
||||
assistant := []byte(`{"items":["original"]}`)
|
||||
correction, err := NewSemanticCorrection(assistant, "Return one corrected replacement.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
assistant[0] = '['
|
||||
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||
t.Fatalf("assistant response = %q, want owned original content", got)
|
||||
}
|
||||
|
||||
clone, err := CloneSemanticCorrection(correction)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneSemanticCorrection() error = %v", err)
|
||||
}
|
||||
clone.AssistantResponse[0] = '['
|
||||
if got := string(correction.AssistantResponse); got != `{"items":["original"]}` {
|
||||
t.Fatalf("source correction changed through clone = %q", got)
|
||||
}
|
||||
if nilClone, err := CloneSemanticCorrection(nil); err != nil || nilClone != nil {
|
||||
t.Fatalf("CloneSemanticCorrection(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(correction)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal correction: %v", err)
|
||||
}
|
||||
if string(encoded) != "{}" {
|
||||
t.Fatalf("correction JSON = %s, want no sensitive content", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionContractsRejectInvalidContent(t *testing.T) {
|
||||
tooLongAssistant := bytes.Repeat([]byte("a"), MaxAssistantResponseBytes+1)
|
||||
tooLongGuidance := strings.Repeat("a", MaxCorrectionGuidanceBytes+1)
|
||||
tooLongReason := strings.Repeat("a", MaxValidationReasonCodeBytes+1)
|
||||
tooLongValidationGuidance := strings.Repeat("a", MaxValidationCorrectionGuidanceBytes+1)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
call func() error
|
||||
}{
|
||||
{"blank assistant", func() error { _, err := NewSemanticCorrection([]byte(" \n"), "guidance"); return err }},
|
||||
{"invalid assistant utf8", func() error { _, err := NewSemanticCorrection([]byte{0xff}, "guidance"); return err }},
|
||||
{"oversized assistant", func() error { _, err := NewSemanticCorrection(tooLongAssistant, "guidance"); return err }},
|
||||
{"blank guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), " \t"); return err }},
|
||||
{"invalid guidance utf8", func() error { _, err := NewSemanticCorrection([]byte("response"), string([]byte{0xff})); return err }},
|
||||
{"oversized guidance", func() error { _, err := NewSemanticCorrection([]byte("response"), tooLongGuidance); return err }},
|
||||
{"unsupported protocol", func() error { _, err := NewModelCandidate([]byte("response"), "multiple_responses"); return err }},
|
||||
{"missing candidate protocol", func() error { _, err := NewModelCandidate([]byte("response"), ""); return err }},
|
||||
{"blank candidate response", func() error { _, err := NewModelCandidate([]byte(" "), CorrectionProtocolSingleResponseV1); return err }},
|
||||
{"missing rejection reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"missing rejection guidance", func() error { return ValidateValidationResult(ValidationResult{ReasonCode: "invalid"}) }},
|
||||
{"oversized reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: tooLongReason, CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"blank reason code", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: " \t", CorrectionGuidance: "Correct the response."})
|
||||
}},
|
||||
{"invalid correction guidance utf8", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})})
|
||||
}},
|
||||
{"oversized correction guidance", func() error {
|
||||
return ValidateValidationResult(ValidationResult{ReasonCode: "invalid", CorrectionGuidance: tooLongValidationGuidance})
|
||||
}},
|
||||
{"invalid diagnostics", func() error {
|
||||
return ValidateValidationResult(ValidationResult{Approved: true, Diagnostics: []ProducerDiagnostic{{}}})
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := test.call(); err == nil {
|
||||
t.Fatal("validation error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelCandidateOwnsValidatedResponse(t *testing.T) {
|
||||
response := []byte(`{"items":["original"]}`)
|
||||
candidate, err := NewModelCandidate(response, CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewModelCandidate() error = %v", err)
|
||||
}
|
||||
response[0] = '['
|
||||
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||
t.Fatalf("candidate response = %q, want owned original content", got)
|
||||
}
|
||||
clone, err := CloneModelCandidate(candidate)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneModelCandidate() error = %v", err)
|
||||
}
|
||||
clone.Response[0] = '['
|
||||
if got := string(candidate.Response); got != `{"items":["original"]}` {
|
||||
t.Fatalf("source candidate changed through clone = %q", got)
|
||||
}
|
||||
if nilClone, err := CloneModelCandidate(nil); err != nil || nilClone != nil {
|
||||
t.Fatalf("CloneModelCandidate(nil) = %#v, %v; want nil, nil", nilClone, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationResultAllowsAbsentOptionalCorrectionFields(t *testing.T) {
|
||||
if err := ValidateValidationResult(ValidationResult{Approved: true}); err != nil {
|
||||
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||
}
|
||||
if err := ValidateValidationResult(ValidationResult{ReasonCode: "invalid-evidence", CorrectionGuidance: "Provide source-backed evidence."}); err != nil {
|
||||
t.Fatalf("ValidateValidationResult() error = %v, want nil", err)
|
||||
}
|
||||
if err := CorrectionProtocol("").Validate(); err == nil {
|
||||
t.Fatal("empty correction protocol validation error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneStructuredCompletionRequestOwnsCorrection(t *testing.T) {
|
||||
correction, err := NewSemanticCorrection([]byte(`{"value":"original"}`), "Correct the value.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
attempts := 2
|
||||
request := StructuredCompletionRequest{
|
||||
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"labels": []string{"original"}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
Correction: correction,
|
||||
}
|
||||
clone, err := CloneStructuredCompletionRequest(request)
|
||||
if err != nil {
|
||||
t.Fatalf("CloneStructuredCompletionRequest() error = %v", err)
|
||||
}
|
||||
correction.AssistantResponse[0] = '['
|
||||
request.Inputs["source"] = NewLLMInputMaterial("source", "application/json", []byte(`{"source":false}`), "", "")
|
||||
*request.StructuredOutputRepairAttempts = 7
|
||||
if got := string(clone.Correction.AssistantResponse); got != `{"value":"original"}` {
|
||||
t.Fatalf("cloned correction response = %q, want owned original content", got)
|
||||
}
|
||||
if got := string(clone.Inputs["source"].Content); got != `{"source":true}` {
|
||||
t.Fatalf("cloned input = %q, want owned original content", got)
|
||||
}
|
||||
if clone.StructuredOutputRepairAttempts == nil || *clone.StructuredOutputRepairAttempts != 2 {
|
||||
t.Fatalf("cloned repair attempts = %v, want 2", clone.StructuredOutputRepairAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredCompletionRequestDebugSummaryOmitsCorrectionContent(t *testing.T) {
|
||||
const assistantResponse = `{"secret":"assistant response"}`
|
||||
const userGuidance = "secret user guidance"
|
||||
correction, err := NewSemanticCorrection([]byte(assistantResponse), userGuidance)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
request := StructuredCompletionRequest{
|
||||
Inputs: LLMInputSet{"source": NewLLMInputMaterial("source", "application/json", []byte(`{"source":true}`), "", "")},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
Correction: correction,
|
||||
}
|
||||
|
||||
summary := request.DebugSummary()
|
||||
if summary.InputCount != 1 || summary.VariableCount != 1 || summary.Correction == nil {
|
||||
t.Fatalf("debug summary = %#v, want input, variable, and correction metadata", summary)
|
||||
}
|
||||
if summary.Correction.AssistantResponseBytes != len(assistantResponse) || summary.Correction.UserGuidanceBytes != len(userGuidance) ||
|
||||
summary.Correction.AssistantResponseDigest == "" || summary.Correction.UserGuidanceDigest == "" {
|
||||
t.Fatalf("correction summary = %#v, want byte counts and digests", summary.Correction)
|
||||
}
|
||||
encoded, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal debug summary: %v", err)
|
||||
}
|
||||
for _, rendered := range []string{string(encoded), fmt.Sprintf("%+v", request), fmt.Sprintf("%#v", request)} {
|
||||
for _, secret := range []string{assistantResponse, userGuidance} {
|
||||
if strings.Contains(rendered, secret) {
|
||||
t.Fatalf("content-safe request rendering leaked %q: %s", secret, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
52
internal/framework/contracts/validation_result.go
Normal file
52
internal/framework/contracts/validation_result.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxValidationReasonCodeBytes = 128
|
||||
MaxValidationCorrectionGuidanceBytes = 4 * 1024
|
||||
)
|
||||
|
||||
func ValidateValidationResult(result ValidationResult) error {
|
||||
if err := ValidateProducerDiagnostics(result.Diagnostics); err != nil {
|
||||
return fmt.Errorf("validation diagnostics: %w", err)
|
||||
}
|
||||
if !result.Approved && result.ReasonCode == "" {
|
||||
return errors.New("validation rejection reason code must not be empty")
|
||||
}
|
||||
if result.ReasonCode != "" {
|
||||
if err := validateBoundedText(result.ReasonCode, MaxValidationReasonCodeBytes, "validation reason code", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !result.Approved && result.CorrectionGuidance == "" {
|
||||
return errors.New("validation rejection correction guidance must not be empty")
|
||||
}
|
||||
if result.CorrectionGuidance != "" {
|
||||
if err := validateBoundedText(result.CorrectionGuidance, MaxValidationCorrectionGuidanceBytes, "validation correction guidance", false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBoundedText(value string, maximum int, name string, optional bool) error {
|
||||
if value == "" && optional {
|
||||
return nil
|
||||
}
|
||||
if !utf8.ValidString(value) {
|
||||
return fmt.Errorf("%s must be valid UTF-8", name)
|
||||
}
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("%s must not be blank", name)
|
||||
}
|
||||
if len(value) > maximum {
|
||||
return fmt.Errorf("%s exceeds maximum length", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
33
internal/framework/contracts/validation_result_test.go
Normal file
33
internal/framework/contracts/validation_result_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package contracts
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidationResultRejectsInvalidFindings(t *testing.T) {
|
||||
for _, result := range []ValidationResult{
|
||||
{CorrectionGuidance: "Correct the response."},
|
||||
{ReasonCode: "invalid"},
|
||||
{ReasonCode: strings.Repeat("a", MaxValidationReasonCodeBytes+1), CorrectionGuidance: "Correct the response."},
|
||||
{ReasonCode: " \t", CorrectionGuidance: "Correct the response."},
|
||||
{ReasonCode: "invalid", CorrectionGuidance: string([]byte{0xff})},
|
||||
{ReasonCode: "invalid", CorrectionGuidance: strings.Repeat("a", MaxValidationCorrectionGuidanceBytes+1)},
|
||||
{Approved: true, Diagnostics: []ProducerDiagnostic{{}}},
|
||||
} {
|
||||
if err := ValidateValidationResult(result); err == nil {
|
||||
t.Fatalf("invalid validation result accepted: %#v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationResultAllowsAbsentOptionalCorrectionFields(t *testing.T) {
|
||||
for _, result := range []ValidationResult{
|
||||
{Approved: true},
|
||||
{ReasonCode: "invalid-evidence", CorrectionGuidance: "Provide source-backed evidence."},
|
||||
} {
|
||||
if err := ValidateValidationResult(result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user