weatherreporter/internal/app/profile_execution.go

package app

import (
	"context"
	"errors"
	"fmt"
	"reflect"

	"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)

type profileExecutionRequest struct {
	Prepared    preparedReport
	Prompt      PromptInspectionResult
	Profile     promptexec.ProfileInspection
	Executor    promptexec.Executor
	DebugWriter *promptdebug.PromptDebugWriter
	DebugRef    *promptdebug.PromptDebugRef
}

type profileExecutionOutcome struct {
	ProfileID        string
	BackendID        string
	ModelName        string
	ValidationStatus promptexec.ValidationStatus
	RepairAttempts   *int
	LLMDebugPath     string
}

type profileExecutionError struct {
	operation       string
	err             error
	callbackFailure bool
}

func (e *profileExecutionError) Error() string {
	return e.operation + ": " + e.err.Error()
}

func (e *profileExecutionError) Unwrap() error {
	return e.err
}

func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) {
	outcome := profileExecutionOutcome{
		ProfileID: req.Profile.ProfileID,
		BackendID: req.Profile.BackendID,
		ModelName: req.Profile.ModelName,
	}
	if req.Executor == nil {
		return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)}
	}
	if err := validatePreparedExecutionRequest(req); err != nil {
		return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: err}
	}

	callbackFailed := false
	preparationCount := 0
	var preparation promptexec.Preparation
	preparationCallback := func(value promptexec.Preparation, debug *promptexec.PreparationDebug) error {
		preparationCount++
		if preparationCount != 1 {
			return promptProvenanceError()
		}
		if err := validatePreparationProvenance(req, value); err != nil {
			return err
		}
		preparation = clonePreparation(value)
		if req.DebugWriter == nil || !req.DebugWriter.Enabled() {
			return nil
		}
		if req.DebugRef == nil {
			callbackFailed = true
			return promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))
		}
		path, err := req.DebugWriter.WritePreparation(*req.DebugRef, value, debug)
		if err != nil {
			callbackFailed = true
			return promptDebugWriteError(err)
		}
		outcome.LLMDebugPath = path
		return nil
	}

	captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled()
	execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
		PromptID:      req.Prompt.PromptID,
		PromptVersion: req.Prompt.PromptVersion,
		ProfileID:     req.Profile.ProfileID,
		DataPackage:   req.Prepared.dataPackageCopy(),
		CaptureDebug:  captureDebug,
	}, preparationCallback)
	if err != nil {
		if callbackFailed {
			return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
		}
		if req.DebugWriter != nil && req.DebugWriter.Enabled() && req.DebugRef != nil {
			var generationError *promptexec.GenerationError
			if errors.As(err, &generationError) {
				path, debugErr := req.DebugWriter.WriteFailure(*req.DebugRef, generationError)
				if path != "" {
					outcome.LLMDebugPath = path
				}
				if debugErr != nil {
					err = errors.Join(err, promptDebugWriteError(debugErr))
				}
			}
		}
		return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
	}
	if execution == nil {
		return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
	}
	outcome.RepairAttempts = repairAttemptsPointer(execution.Validation.RepairAttempts)
	if preparationCount != 1 {
		return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: promptProvenanceError()}
	}
	if err := validateExecutionProvenance(req, preparation, *execution); err != nil {
		return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: err}
	}
	outcome.ValidationStatus = execution.Validation.Status
	if err := generatedtext.ValidateRawOutput(execution.RawOutput); err != nil {
		return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
	}
	if req.DebugWriter != nil && req.DebugWriter.Enabled() {
		if req.DebugRef == nil {
			return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))}
		}
		path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution)
		if err != nil {
			return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)}
		}
		if path != "" {
			outcome.LLMDebugPath = path
		}
	}

	if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
		return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)}
	}
	if execution.Validation.Status == promptexec.ValidationFailed {
		return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)}
	}

	generatedText, err := req.Prepared.handler.Validate(execution.RawOutput)
	if err != nil {
		return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
	}
	identity, snapshot, derived, err := req.Prepared.renderInputs()
	if err != nil {
		return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err}
	}
	renderContext, err := req.Prepared.handler.BuildRenderContext(identity, snapshot, derived, generatedText)
	if err != nil {
		return outcome, nil, &profileExecutionError{operation: "build render context", err: err}
	}
	rendered, err := req.Prepared.handler.Render(renderContext)
	if err != nil {
		return outcome, nil, &profileExecutionError{operation: "render template", err: err}
	}
	return outcome, rendered, nil
}

func validatePreparedExecutionRequest(req profileExecutionRequest) error {
	definition := req.Prepared.resolved.Definition
	if definition.PromptID != req.Prompt.PromptID || definition.PromptVersion != req.Prompt.PromptVersion ||
		definition.GeneratedTextSchemaID != req.Prepared.handler.SchemaID() {
		return promptProvenanceError()
	}
	if req.Prompt.ProfileID != "" && (req.Prompt.ProfileID != req.Profile.ProfileID || req.Prompt.BackendID != req.Profile.BackendID || req.Prompt.ModelName != req.Profile.ModelName) {
		return promptProvenanceError()
	}
	if req.Prompt.PromptHash == "" || req.Profile.ProfileID == "" || req.Profile.ModelName == "" {
		return promptProvenanceError()
	}
	return nil
}

func validatePreparationProvenance(req profileExecutionRequest, preparation promptexec.Preparation) error {
	definition := req.Prepared.resolved.Definition
	if preparation.PromptID != req.Prompt.PromptID || preparation.PromptVersion != req.Prompt.PromptVersion || preparation.PromptHash != req.Prompt.PromptHash ||
		preparation.ProfileID != req.Profile.ProfileID || preparation.BackendID != req.Profile.BackendID || preparation.ModelName != req.Profile.ModelName ||
		!validPromptOutput(definition, preparation.Output) {
		return promptProvenanceError()
	}
	return nil
}

func validateExecutionProvenance(req profileExecutionRequest, preparation promptexec.Preparation, execution promptexec.Execution) error {
	definition := req.Prepared.resolved.Definition
	if execution.PromptID != preparation.PromptID || execution.PromptVersion != preparation.PromptVersion || execution.PromptHash != preparation.PromptHash ||
		execution.RenderedPromptHash != preparation.RenderedPromptHash || !reflect.DeepEqual(execution.InputHashes, preparation.InputHashes) ||
		execution.ProfileID != preparation.ProfileID || execution.BackendID != preparation.BackendID || execution.ModelName != preparation.ModelName ||
		execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" ||
		execution.Validation.RepairAttempts < 0 || execution.Validation.RepairAttempts > preparation.Output.RepairAttempts ||
		preparation.Output.RepairAttempts != definition.GeneratedTextRepairAttempts {
		return promptProvenanceError()
	}
	return nil
}

func repairAttemptsPointer(value int) *int {
	copy := value
	return &copy
}

func promptProvenanceError() error {
	return promptexec.NewError(promptexec.InvalidConfiguration, "prompt execution provenance is inconsistent", nil)
}

func clonePreparation(value promptexec.Preparation) promptexec.Preparation {
	if value.InputHashes != nil {
		inputHashes := make(map[string]string, len(value.InputHashes))
		for name, hash := range value.InputHashes {
			inputHashes[name] = hash
		}
		value.InputHashes = inputHashes
	}
	return value
}