All checks were successful
ci/woodpecker/tag/release Pipeline was successful
322 lines
12 KiB
Go
322 lines
12 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/responseschema"
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/structuredoutput"
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/prompts"
|
|
pk "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
// CallHooks receive one logical start and finish record and the execution
|
|
// events associated with the call.
|
|
type CallHooks struct {
|
|
OnCall func(structuredoutput.CallRecord)
|
|
OnEvent pk.ExecutionObserver
|
|
OnEventWithCall func(uint64, structuredoutput.DiagnosticContext, pk.ExecutionEvent)
|
|
// OnProgress runs outside the engine's timed operation while the caller
|
|
// context is active. A nil count marks the start.
|
|
OnProgress func(structuredoutput.DiagnosticContext, *outcomes.ExecutionCounts, error)
|
|
}
|
|
|
|
type RoleClient struct {
|
|
runtime *Runtime
|
|
role string
|
|
profileID string
|
|
settings config.LLM
|
|
hooks CallHooks
|
|
}
|
|
|
|
var _ structuredoutput.Client = (*RoleClient)(nil)
|
|
|
|
func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hooks CallHooks) (*RoleClient, error) {
|
|
if r == nil || r.Engine == nil {
|
|
return nil, fmt.Errorf("LLM engine is unavailable")
|
|
}
|
|
if profileID == "" {
|
|
return nil, fmt.Errorf("profile selection is required")
|
|
}
|
|
return &RoleClient{runtime: r, role: role, profileID: profileID, settings: settings, hooks: hooks}, nil
|
|
}
|
|
|
|
type pendingCall struct {
|
|
request pk.RunRequest
|
|
once sync.Once
|
|
complete func(*pk.RunResult, error) (structuredoutput.Response, error)
|
|
response structuredoutput.Response
|
|
err error
|
|
}
|
|
|
|
func (p *pendingCall) finish(result *pk.RunResult, err error) (structuredoutput.Response, error) {
|
|
p.once.Do(func() {
|
|
p.response, p.err = p.complete(result, err)
|
|
})
|
|
return p.response, p.err
|
|
}
|
|
|
|
func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) {
|
|
var mu sync.Mutex
|
|
var terminal pk.ExecutionEvent
|
|
var terminalAvailable bool
|
|
var prepared *structuredoutput.PreparedSnapshot
|
|
var prompt structuredoutput.PromptMetadata
|
|
var record structuredoutput.CallRecord
|
|
progressContext := req.Context
|
|
progressContext.Role = c.role
|
|
emitProgress := func(counts *outcomes.ExecutionCounts, err error) {
|
|
if progress := c.hooks.OnProgress; progress != nil && parent.Err() == nil {
|
|
progress(progressContext, counts, err)
|
|
}
|
|
}
|
|
emitProgress(nil, nil)
|
|
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
|
record.CallID = c.runtime.nextCallID.Add(1)
|
|
record.RequestedPromptID = req.PromptID
|
|
record.Context = copyContext(req.Context)
|
|
record.Context.Role = c.role
|
|
}
|
|
if onCall := c.hooks.OnCall; onCall != nil {
|
|
record.Started = time.Now().UTC()
|
|
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
|
|
onCall(record)
|
|
}
|
|
call := &pendingCall{complete: func(result *pk.RunResult, runErr error) (structuredoutput.Response, error) {
|
|
var response structuredoutput.Response
|
|
mu.Lock()
|
|
if terminalAvailable {
|
|
response.Execution = outcomes.ExecutionCounts{
|
|
Available: true, ProviderAttempts: int(terminal.Attempts),
|
|
TransientRetries: int(terminal.TransientRetries), StructuralRepairs: int(terminal.RepairCalls),
|
|
RetriesExhausted: terminal.RetriesExhausted,
|
|
}
|
|
if terminal.StatusCode != 0 {
|
|
status := terminal.StatusCode
|
|
response.Execution.HTTPStatus = &status
|
|
}
|
|
}
|
|
response.Prepared = prepared
|
|
response.Prompt = prompt
|
|
response.Model = prompt.Model
|
|
mu.Unlock()
|
|
if result != nil {
|
|
if req.CaptureDiagnostics {
|
|
response.FinalContent = result.RawOutput
|
|
}
|
|
response.Model = result.ModelName
|
|
}
|
|
err := c.finishResponse(parent, &response, result, runErr, out)
|
|
if onCall := c.hooks.OnCall; onCall != nil {
|
|
if response.Prompt.PromptID != "" {
|
|
record.Prompt = response.Prompt
|
|
}
|
|
record.Finished = time.Now().UTC()
|
|
if err != nil {
|
|
record.Failure = structuredoutput.FailureRuntime
|
|
if structuredoutput.IsMalformedError(err) {
|
|
record.Failure = structuredoutput.FailureMalformedOutput
|
|
}
|
|
}
|
|
onCall(record)
|
|
}
|
|
emitProgress(&response.Execution, err)
|
|
return response, err
|
|
}}
|
|
fail := func(err error) (*pendingCall, error) {
|
|
_, err = call.finish(nil, err)
|
|
return call, err
|
|
}
|
|
if err := parent.Err(); err != nil {
|
|
return fail(err)
|
|
}
|
|
if err := validateOutputTarget(out); err != nil {
|
|
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err})
|
|
}
|
|
contract, err := OutputContract(req.Schema, c.settings.RepairAttempts)
|
|
if err != nil {
|
|
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err})
|
|
}
|
|
call.request = pk.RunRequest{
|
|
PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables),
|
|
CaptureExchange: req.CaptureExchange,
|
|
TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout,
|
|
Validation: contract,
|
|
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
|
|
if event.Kind == pk.ExecutionFinished {
|
|
mu.Lock()
|
|
terminal = event
|
|
terminalAvailable = true
|
|
mu.Unlock()
|
|
}
|
|
if c.hooks.OnEventWithCall != nil {
|
|
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
|
}
|
|
}),
|
|
}
|
|
if req.CaptureDiagnostics || c.hooks.OnCall != nil {
|
|
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
|
|
metadata := promptMetadata(details)
|
|
var snapshot *structuredoutput.PreparedSnapshot
|
|
if req.CaptureDiagnostics {
|
|
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
|
|
}
|
|
mu.Lock()
|
|
prepared, prompt = snapshot, metadata
|
|
mu.Unlock()
|
|
}
|
|
}
|
|
return call, nil
|
|
}
|
|
|
|
func (c *RoleClient) finishResponse(parent context.Context, response *structuredoutput.Response, result *pk.RunResult, runErr error, out any) error {
|
|
if runErr != nil || parent.Err() != nil {
|
|
var typed *structuredoutput.Failure
|
|
if parent.Err() == nil && errors.As(runErr, &typed) {
|
|
return runErr
|
|
}
|
|
failure := classifiedFailure(parent, runErr, result)
|
|
if failure.Reason == outcomes.ReasonSetupFailure && errors.Is(failure, pk.ErrAPIKeyEnvMissing) {
|
|
failure.Message = fmt.Sprintf("structured model setup failed for %s profile %q: %v", c.role, c.profileID, failure.Cause)
|
|
}
|
|
return failure
|
|
}
|
|
if result == nil {
|
|
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
|
|
}
|
|
if !result.Validation.IsValid {
|
|
return classifiedFailure(parent, nil, result)
|
|
}
|
|
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
|
var typeError *json.UnmarshalTypeError
|
|
if errors.As(decodeErr, &typeError) {
|
|
return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
|
|
}
|
|
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
|
}
|
|
if result.Exchange != nil {
|
|
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
|
|
}
|
|
response.Content = json.RawMessage(result.RawOutput)
|
|
return nil
|
|
}
|
|
|
|
func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
|
call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent)
|
|
if err != nil {
|
|
return call.response, err
|
|
}
|
|
result, err := c.runtime.Engine.Run(ctx, call.request)
|
|
return call.finish(result, err)
|
|
}
|
|
|
|
// OutputContract maps an Audita domain schema to the native request contract.
|
|
func OutputContract(schema responseschema.Key, repairs int) (*pk.OutputContract, error) {
|
|
path, ok := responseschema.AssetPath(schema)
|
|
if !ok {
|
|
return nil, fmt.Errorf("schema %q", schema)
|
|
}
|
|
return &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: path, RepairAttempts: repairs}, nil
|
|
}
|
|
|
|
// classifiedFailure deliberately checks setup defects before broad PromptKit
|
|
// validation/generation wrappers. Provider text is kept only in the typed cause.
|
|
func classifiedFailure(parent context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
|
|
failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause}
|
|
if parent.Err() != nil {
|
|
failure.Class = structuredoutput.FailureCanceled
|
|
failure.Reason = outcomes.ReasonCanceled
|
|
failure.Message = "processing canceled"
|
|
if !errors.Is(cause, parent.Err()) {
|
|
failure.Cause = errors.Join(parent.Err(), cause)
|
|
}
|
|
return failure
|
|
}
|
|
if errors.Is(cause, pk.ErrOperationTimeout) {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonOperationTimeout
|
|
failure.Message = "model operation timed out"
|
|
return failure
|
|
}
|
|
if isSetupFailure(cause) {
|
|
failure.Reason = outcomes.ReasonSetupFailure
|
|
failure.Message = "structured model setup failed"
|
|
return failure
|
|
}
|
|
if errors.Is(cause, pk.ErrTransport) {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonTransportFailed
|
|
failure.Message = "model transport failed"
|
|
return failure
|
|
}
|
|
if errors.Is(cause, context.DeadlineExceeded) {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonOperationTimeout
|
|
failure.Message = "model operation timed out"
|
|
return failure
|
|
}
|
|
if errors.Is(cause, pk.ErrCapacityExceeded) {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonCapacityExceeded
|
|
failure.Message = "model capacity was unavailable"
|
|
return failure
|
|
}
|
|
if errors.Is(cause, pk.ErrMalformedResponse) {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonProviderResponseInvalid
|
|
failure.Message = "model provider response was invalid"
|
|
return failure
|
|
}
|
|
var provider *pk.GenerationError
|
|
if errors.As(cause, &provider) && provider.StatusCode() != 0 {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonProviderFailed
|
|
failure.Message = "model provider failed"
|
|
return failure
|
|
}
|
|
if result != nil && !result.Validation.IsValid {
|
|
failure.Class = structuredoutput.FailureRecoverable
|
|
failure.Reason = outcomes.ReasonInvalidOutputExhausted
|
|
failure.Message = "structured output remained invalid"
|
|
return failure
|
|
}
|
|
if errors.Is(cause, pk.ErrValidation) {
|
|
failure.Reason = outcomes.ReasonSetupFailure
|
|
failure.Message = "structured validation contract failed"
|
|
}
|
|
return failure
|
|
}
|
|
|
|
func isSetupFailure(err error) bool {
|
|
return errors.Is(err, pk.ErrInvalidConfig) || errors.Is(err, pk.ErrInvalidRequest) ||
|
|
errors.Is(err, pk.ErrPromptNotFound) || errors.Is(err, pk.ErrProfileNotFound) ||
|
|
errors.Is(err, pk.ErrPromptLoad) || errors.Is(err, pk.ErrProfileLoad) ||
|
|
errors.Is(err, pk.ErrArtifactLoad) || errors.Is(err, pk.ErrPromptRender)
|
|
}
|
|
|
|
func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
|
return structuredoutput.PromptMetadata{
|
|
PromptID: details.PromptID,
|
|
DefinitionHash: details.PromptHash, RenderedHash: details.RenderedPromptHash,
|
|
ProfileID: details.SelectedProfileID, BackendID: details.SelectedBackendID,
|
|
Model: details.EffectiveModelParams.Model,
|
|
}
|
|
}
|
|
|
|
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
|
|
result := make([]structuredoutput.Message, len(messages))
|
|
for i, message := range messages {
|
|
result[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content}
|
|
if cache := message.CacheControl; cache != nil {
|
|
result[i].CacheControl = &structuredoutput.CacheControl{Type: string(cache.Type), TTL: cache.TTL}
|
|
}
|
|
}
|
|
return result
|
|
}
|