Capture native execution facts and complete scoped debug evidence
This commit is contained in:
255
internal/framework/pipeline/native_capture.go
Normal file
255
internal/framework/pipeline/native_capture.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// NativeCapture retains only one request's prepared snapshot. Events are written
|
||||
// incrementally, not accumulated as a second execution/usage ledger. It never
|
||||
// prepares or executes a request itself.
|
||||
type NativeCapture struct {
|
||||
writer contracts.EvidenceWriter
|
||||
mu sync.Mutex
|
||||
prepared *promptkit.PreparedRun
|
||||
runID string
|
||||
events uint64
|
||||
err error
|
||||
}
|
||||
|
||||
// CaptureNativeRequest composes existing hooks and enables native exchange
|
||||
// capture only when evidence was requested. Request credentials remain excluded
|
||||
// by the native request's own JSON representation.
|
||||
func CaptureNativeRequest(request promptkit.RunRequest, writer contracts.EvidenceWriter) (promptkit.RunRequest, *NativeCapture) {
|
||||
if writer == nil {
|
||||
return request, nil
|
||||
}
|
||||
capture := &NativeCapture{writer: writer}
|
||||
capture.remember(writer.WriteJSON("request.json", request))
|
||||
request.CaptureExchange = true
|
||||
existing := request.OnPrepared
|
||||
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
|
||||
// No serialization or filesystem I/O while native admission is held.
|
||||
// Native hands off a detached value. Copy mutable fields only when an
|
||||
// existing callback also owns that value and may change it.
|
||||
if existing != nil {
|
||||
capture.RetainPrepared(copyPreparedEvidence(details))
|
||||
existing(ctx, details)
|
||||
} else {
|
||||
capture.RetainPrepared(details)
|
||||
}
|
||||
}
|
||||
request.Observer = promptkit.ComposeObservers(request.Observer, capture.observe)
|
||||
return request, capture
|
||||
}
|
||||
|
||||
// RetainPrepared uses the already available split-preparation details on cache
|
||||
// paths, including hits on which OnPrepared and native events never occur.
|
||||
func (capture *NativeCapture) RetainPrepared(details promptkit.PreparedRun) {
|
||||
if capture == nil {
|
||||
return
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
capture.prepared = &details
|
||||
}
|
||||
|
||||
func (capture *NativeCapture) remember(err error) {
|
||||
if capture == nil || err == nil {
|
||||
return
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
if capture.err == nil {
|
||||
capture.err = err
|
||||
}
|
||||
}
|
||||
|
||||
func (capture *NativeCapture) observe(event promptkit.ExecutionEvent) {
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
capture.runID = event.RunID
|
||||
capture.events++
|
||||
err := capture.writer.WriteJSON(path.Join("native", event.RunID, "events", fmt.Sprintf("%06d.json", capture.events)), event)
|
||||
if capture.err == nil {
|
||||
capture.err = err
|
||||
}
|
||||
}
|
||||
|
||||
// Err exposes synchronous callback/write failures at the next normal boundary.
|
||||
func (capture *NativeCapture) Err() error {
|
||||
if capture == nil {
|
||||
return nil
|
||||
}
|
||||
capture.mu.Lock()
|
||||
defer capture.mu.Unlock()
|
||||
if capture.err == nil {
|
||||
return nil
|
||||
}
|
||||
return &contracts.CaptureError{Cause: capture.err}
|
||||
}
|
||||
|
||||
// Finish records available evidence even when native execution returned no
|
||||
// result. Unassessed marks a response withheld from the application by native
|
||||
// coordination; cacheHit never fabricates an execution or historical result.
|
||||
func (capture *NativeCapture) Finish(result *promptkit.RunResult, nativeErr error, unassessed, cacheHit bool) error {
|
||||
if capture == nil {
|
||||
return nil
|
||||
}
|
||||
capture.mu.Lock()
|
||||
prepared, runID := capture.prepared, capture.runID
|
||||
capture.prepared = nil
|
||||
capture.mu.Unlock()
|
||||
if result != nil {
|
||||
runID = result.RunID
|
||||
}
|
||||
prefix := "unexecuted"
|
||||
if runID != "" {
|
||||
prefix = path.Join("native", runID)
|
||||
}
|
||||
writeJSON := func(name string, value any) {
|
||||
capture.remember(capture.writer.WriteJSON(path.Join(prefix, name), value))
|
||||
}
|
||||
writeText := func(name, value string) {
|
||||
capture.remember(capture.writer.WriteBytes(path.Join(prefix, name), []byte(value)))
|
||||
}
|
||||
if prepared != nil {
|
||||
writeJSON("prepared.json", prepared)
|
||||
writeText("prepared-messages.txt", readableNativeMessages("Prepared request messages", prepared.Messages))
|
||||
}
|
||||
if result != nil {
|
||||
writeJSON("result.json", result)
|
||||
writeText("response.txt", result.RawOutput)
|
||||
if result.Exchange != nil {
|
||||
details := result.Exchange.Details()
|
||||
writeJSON("exchange.json", details)
|
||||
writeText("messages.txt", readableNativeMessages("Completed exchange", details.Messages))
|
||||
}
|
||||
}
|
||||
errorText := ""
|
||||
if nativeErr != nil {
|
||||
errorText = nativeErr.Error()
|
||||
}
|
||||
writeJSON("availability.json", map[string]any{
|
||||
"run_id": runID, "prepared_available": prepared != nil,
|
||||
"result_available": result != nil, "unassessed": unassessed,
|
||||
"cache_hit": cacheHit, "error": errorText,
|
||||
})
|
||||
return capture.Err()
|
||||
}
|
||||
|
||||
func readableNativeMessages(label string, messages []promptkit.RenderedMessage) string {
|
||||
var text strings.Builder
|
||||
text.WriteString(label + "\n")
|
||||
for _, message := range messages {
|
||||
text.WriteString("\n[" + message.Role + "]\n")
|
||||
text.WriteString(message.Content)
|
||||
text.WriteByte('\n')
|
||||
}
|
||||
return text.String()
|
||||
}
|
||||
|
||||
type scopedEvidence struct {
|
||||
writer contracts.EvidenceWriter
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (scope scopedEvidence) WriteJSON(name string, value any) error {
|
||||
return scope.writer.WriteJSON(path.Join(scope.prefix, name), value)
|
||||
}
|
||||
func (scope scopedEvidence) WriteBytes(name string, value []byte) error {
|
||||
return scope.writer.WriteBytes(path.Join(scope.prefix, name), value)
|
||||
}
|
||||
func scopeEvidence(writer contracts.EvidenceWriter, prefix string) contracts.EvidenceWriter {
|
||||
if writer == nil {
|
||||
return nil
|
||||
}
|
||||
return scopedEvidence{writer: writer, prefix: prefix}
|
||||
}
|
||||
|
||||
func operationExecution(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk) contracts.ExecutionOptions {
|
||||
options := input.Execution
|
||||
writer := options.Capture
|
||||
if writer == nil && input.Debug != nil && input.Debug.Enabled() {
|
||||
writer = input.Debug
|
||||
}
|
||||
prefix := "operations"
|
||||
if origin.StepID != "" {
|
||||
prefix = path.Join(prefix, "steps", fileio.EncodePathComponent(origin.StepID))
|
||||
}
|
||||
prefix = path.Join(prefix, string(origin.Stage))
|
||||
if origin.LaneID != "" {
|
||||
prefix = path.Join(prefix, "lanes", fileio.EncodePathComponent(origin.LaneID))
|
||||
}
|
||||
if chunk != nil {
|
||||
prefix = path.Join(prefix, "chunks", fileio.EncodePathComponent(chunk.ID))
|
||||
}
|
||||
prefix = path.Join(prefix, "modules", fileio.EncodePathComponent(origin.ModuleKey))
|
||||
options.Capture = scopeEvidence(writer, prefix)
|
||||
return options
|
||||
}
|
||||
|
||||
// Native schema/extra-parameter documents are JSON values. Copy their mutable
|
||||
// containers, without serializing large message bodies in OnPrepared.
|
||||
func copyEvidenceJSON(value any) any {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
copy := make(map[string]any, len(value))
|
||||
for key, item := range value {
|
||||
copy[key] = copyEvidenceJSON(item)
|
||||
}
|
||||
return copy
|
||||
case []any:
|
||||
copy := make([]any, len(value))
|
||||
for index, item := range value {
|
||||
copy[index] = copyEvidenceJSON(item)
|
||||
}
|
||||
return copy
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func copyPreparedEvidence(details promptkit.PreparedRun) promptkit.PreparedRun {
|
||||
details.InputHashes = maps.Clone(details.InputHashes)
|
||||
if details.EffectiveModelParams.ExtraParams != nil {
|
||||
details.EffectiveModelParams.ExtraParams = copyEvidenceJSON(details.EffectiveModelParams.ExtraParams).(map[string]any)
|
||||
}
|
||||
details.OutputContract.Schema = copyEvidenceJSON(details.OutputContract.Schema)
|
||||
if details.StructuredOutput != nil {
|
||||
structured := *details.StructuredOutput
|
||||
if structured.JSONSchema != nil {
|
||||
schema := *structured.JSONSchema
|
||||
schema.Schema = copyEvidenceJSON(schema.Schema)
|
||||
structured.JSONSchema = &schema
|
||||
}
|
||||
details.StructuredOutput = &structured
|
||||
}
|
||||
messages := make([]promptkit.RenderedMessage, len(details.Messages))
|
||||
for index, message := range details.Messages {
|
||||
messages[index] = message
|
||||
if message.CacheControl != nil {
|
||||
hint := *message.CacheControl
|
||||
messages[index].CacheControl = &hint
|
||||
}
|
||||
}
|
||||
details.Messages = messages
|
||||
return details
|
||||
}
|
||||
|
||||
func captureFailure(primary, recording error) error {
|
||||
if recording == nil {
|
||||
return primary
|
||||
}
|
||||
return errors.Join(primary, recording)
|
||||
}
|
||||
Reference in New Issue
Block a user