Files
notarius/internal/framework/pipeline/debug.go

265 lines
8.4 KiB
Go

package pipeline
import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type DebugRecorder interface {
Enabled() bool
WriteJSON(name string, payload any) error
WriteBytes(name string, data []byte) error
}
type noopDebugRecorder struct{}
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
func (noopDebugRecorder) Enabled() bool { return false }
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
type debugTimedEnvelope struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
Error string `json:"error,omitempty"`
}
type debugBinaryEnvelope struct {
Text string `json:"text,omitempty"`
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw debugBinaryEnvelope `json:"raw,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceDocument struct {
ID string `json:"id"`
Kind string `json:"kind"`
Format string `json:"format,omitempty"`
Digest string `json:"digest,omitempty"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref source.SourceRef `json:"ref"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Annotations source.ChunkAnnotations `json:"annotations,omitempty"`
PlanAnnotations source.ChunkAnnotations `json:"plan_annotations,omitempty"`
}
type debugChunkPlan struct {
SourceDigest string `json:"source_digest,omitempty"`
Ranges []debugChunkRange `json:"ranges,omitempty"`
Annotations map[string]any `json:"annotations,omitempty"`
}
type debugChunkRange struct {
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
Annotations map[string]any `json:"annotations,omitempty"`
}
type debugSerializedOutput struct {
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content debugBinaryEnvelope `json:"content"`
}
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
if recorder == nil || !recorder.Enabled() {
return nil
}
if envelope.CompletedAt.IsZero() {
envelope.CompletedAt = time.Now().UTC()
}
if envelope.StartedAt.IsZero() {
envelope.StartedAt = envelope.CompletedAt
}
if envelope.DurationMS == 0 {
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
}
return recorder.WriteJSON(name, envelope)
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
text := ""
if utf8.Valid(content) {
text = string(content)
}
return debugBinaryEnvelope{
Text: text,
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: debugContentDigest(content),
MediaType: mediaType,
Metadata: metadata,
}
}
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
if doc == nil {
return nil
}
return &debugSourceDocument{
ID: doc.ID,
Kind: doc.Kind,
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnitsForDebug(doc.Units),
Metadata: doc.Metadata,
}
}
func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
return debugSourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnitsForDebug(chunk.Units),
Metadata: chunk.Metadata,
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
}
}
func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
if len(units) == 0 {
return nil
}
cloned := make([]source.SourceUnit, len(units))
for i, unit := range units {
cloned[i] = source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: unit.Metadata,
}
}
return cloned
}
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil
}
out := make([]debugSourceChunk, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, debugSourceChunkEnvelope(chunk))
}
return out
}
func debugChunkPlanEnvelope(plan source.ChunkPlan) debugChunkPlan {
envelope := debugChunkPlan{
SourceDigest: plan.SourceDigest,
Ranges: make([]debugChunkRange, len(plan.Ranges)),
Annotations: debugChunkAnnotations(plan.Annotations),
}
for i, chunkRange := range plan.Ranges {
envelope.Ranges[i] = debugChunkRange{
StartUnitID: chunkRange.StartUnitID,
EndUnitID: chunkRange.EndUnitID,
Annotations: debugChunkAnnotations(chunkRange.Annotations),
}
}
return envelope
}
func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
if len(annotations) == 0 {
return nil
}
out := make(map[string]any, len(annotations))
for namespace, raw := range annotations {
if json.Valid(raw) {
out[namespace] = append(json.RawMessage(nil), raw...)
} else {
out[namespace] = string(raw)
}
}
return out
}
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema)
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(output.Artifact.Content)
return debugSerializedOutput{
StepID: output.StepID, LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: digest,
Content: content,
}
}
func debugSerializedOutputEnvelopes(outputs []contracts.SerializedOutput) []debugSerializedOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugSerializedOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugSerializedOutputEnvelope(output))
}
return out
}
type debugOutputFile struct {
Name string `json:"name"`
ContentType string `json:"content_type,omitempty"`
Content debugBinaryEnvelope `json:"content"`
}
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
if len(files) == 0 {
return nil
}
out := make([]debugOutputFile, 0, len(files))
for _, file := range files {
out = append(out, debugOutputFile{
Name: file.Name,
ContentType: file.ContentType,
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
})
}
return out
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}