notarius/internal/framework/pipeline/validation_executor.go

package pipeline

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)

type validationOutcome string

const (
	validationApproved validationOutcome = "approved"
	validationRejected validationOutcome = "rejected"
	validationFailed   validationOutcome = "failed"
	validationSkipped  validationOutcome = "skipped"
)

const correctionRequestIntroduction = "The previous response failed semantic validation. Return one complete corrected replacement response, not a patch, explanation, or commentary.\n\nCorrect all of the following:\n"

// validationRecord captures the settled result of one configured validator.
// Its fields remain private so reports cannot expose mutable diagnostic storage.
type validationRecord struct {
	validatorName      string
	outcome            validationOutcome
	attemptCount       int
	reasonCode         string
	message            string
	diagnosticPath     string
	diagnostics        []contracts.ProducerDiagnostic
	correctionGuidance string
	failure            error
}

func (record validationRecord) clone() validationRecord {
	record.diagnostics = contracts.CloneProducerDiagnostics(record.diagnostics)
	return record
}

// validationReport is the immutable, ordered result of one validator chain.
type validationReport struct {
	records []validationRecord
}

func (report validationReport) Records() []validationRecord {
	records := make([]validationRecord, len(report.records))
	for index, record := range report.records {
		records[index] = record.clone()
	}
	return records
}

func (report validationReport) Diagnostics() []validationDiagnosticRecord {
	var diagnostics []validationDiagnosticRecord
	for _, record := range report.records {
		if record.outcome != validationApproved && record.outcome != validationRejected {
			continue
		}
		for _, diagnostic := range record.diagnostics {
			diagnostics = append(diagnostics, validationDiagnosticRecord{validatorName: record.validatorName, diagnostic: cloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic})[0]})
		}
	}
	return diagnostics
}

type validationDiagnosticRecord struct {
	validatorName string
	diagnostic    contracts.ProducerDiagnostic
}

func (report validationReport) FirstRejection() *validationRecord {
	for _, record := range report.records {
		if record.outcome == validationRejected {
			clone := record.clone()
			return &clone
		}
	}
	return nil
}

func (report validationReport) FirstFailure() *validationRecord {
	for _, record := range report.records {
		if record.outcome == validationFailed {
			clone := record.clone()
			return &clone
		}
	}
	return nil
}

func (report validationReport) CorrectionRequest() (string, error) {
	seen := make(map[string]struct{})
	parts := make([]string, 0, len(report.records))
	for _, record := range report.records {
		if record.outcome != validationRejected {
			continue
		}
		guidance := strings.TrimSpace(record.correctionGuidance)
		if guidance == "" {
			return "", fmt.Errorf("validator %q rejected output without correction guidance", record.validatorName)
		}
		if _, exists := seen[guidance]; exists {
			continue
		}
		seen[guidance] = struct{}{}
		parts = append(parts, guidance)
	}
	if len(parts) == 0 {
		return "", errors.New("validation report contains no correction guidance")
	}
	var builder strings.Builder
	builder.WriteString(correctionRequestIntroduction)
	for index, guidance := range parts {
		fmt.Fprintf(&builder, "%d. %s", index+1, guidance)
		if index+1 < len(parts) {
			builder.WriteByte('\n')
		}
	}
	request := builder.String()
	if len(request) > contracts.MaxCorrectionGuidanceBytes {
		return "", fmt.Errorf("aggregate correction request exceeds maximum length of %d bytes", contracts.MaxCorrectionGuidanceBytes)
	}
	return request, nil
}

type validationInvocation struct {
	result  contracts.ValidationResult
	skipped bool
	reason  string
	message string
}

func skippedValidation(reason, message string) validationInvocation {
	return validationInvocation{skipped: true, reason: strings.TrimSpace(reason), message: strings.TrimSpace(message)}
}

type validationInvoker func(context.Context, preparedValidator, int) (validationInvocation, error)

type validationFrameworkError struct{ err error }

func (err validationFrameworkError) Error() string { return err.err.Error() }

func (err validationFrameworkError) Unwrap() error { return err.err }

func fatalValidationError(err error) error {
	if err == nil {
		return nil
	}
	return validationFrameworkError{err: err}
}

func executeValidationChain(ctx context.Context, chain preparedValidatorChain, invoke validationInvoker) (validationReport, error) {
	if ctx == nil {
		return validationReport{}, errors.New("validator execution context must not be nil")
	}
	if invoke == nil {
		return validationReport{}, errors.New("validator invocation must not be nil")
	}
	report := validationReport{records: make([]validationRecord, 0, len(chain.validators))}
	for index, validator := range chain.validators {
		validator.position = index + 1
		if err := ctx.Err(); err != nil {
			return validationReport{}, err
		}
		binding := validator.resolved.Binding
		attemptLimit := 1
		if validator.resolved.ExecutionClass == contracts.ExecutionClassLLMBacked {
			attemptLimit += binding.Retries
		}
		for attempt := 1; attempt <= attemptLimit; attempt++ {
			if err := ctx.Err(); err != nil {
				return validationReport{}, err
			}
			invocation, err := invoke(ctx, validator, attempt)
			if err != nil {
				var frameworkErr validationFrameworkError
				if errors.As(err, &frameworkErr) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
					return validationReport{}, err
				}
				if attempt == attemptLimit {
					report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator failed after retry exhaustion", failure: err})
				}
				continue
			}
			if invocation.skipped {
				report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationSkipped, attemptCount: attempt, reasonCode: invocation.reason, message: invocation.message})
				break
			}
			if err := contracts.ValidateProducerDiagnostics(invocation.result.Diagnostics); err != nil {
				return validationReport{}, fatalValidationError(fmt.Errorf("validator %q returned invalid diagnostics: %w", binding.Module, err))
			}
			if err := contracts.ValidateValidationResult(invocation.result); err != nil {
				if attempt == attemptLimit {
					report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationFailed, attemptCount: attempt, message: "validator returned an invalid result", failure: err})
				}
				continue
			}
			if invocation.result.Approved {
				report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationApproved, attemptCount: attempt, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), diagnosticPath: invocation.result.DiagnosticArtifactPath})
				break
			}
			message := invocation.result.Message
			if message == "" {
				message = "output rejected"
			}
			report.records = append(report.records, validationRecord{validatorName: binding.Module, outcome: validationRejected, attemptCount: attempt, reasonCode: invocation.result.ReasonCode, message: message, diagnosticPath: invocation.result.DiagnosticArtifactPath, diagnostics: cloneProducerDiagnostics(invocation.result.Diagnostics), correctionGuidance: invocation.result.CorrectionGuidance})
			break
		}
	}
	return report, nil
}

func validatorFailureError(record validationRecord) error {
	if record.failure != nil {
		return record.failure
	}
	return fmt.Errorf("validator %q failed after %d attempt(s)", record.validatorName, record.attemptCount)
}

func validatorAttemptPath(base string, attempt int) string {
	if attempt == 1 {
		return base
	}
	return fmt.Sprintf("%s-validator-%02d", base, attempt)
}