Files
notarius/internal/cli/pipeline_commands_test.go

277 lines
13 KiB
Go
Raw Normal View History

package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
const inspectionConfiguration = `version: 6
output: {directory: output-must-not-exist}
cache_directory: cache-must-not-exist
debug: {directory: debug-must-not-exist}
pipelines:
demo:
input: {module: fixture/input}
chunk: {module: fixture/chunk}
output: {module: fixture/output}
steps:
- id: build
artifacts:
notes:
extract:
module: fixture/extract
references: {context: {path: reference-must-not-exist.txt}}
validators: [{module: fixture/validator}]
merge: {module: fixture/merge}
normalize: {module: fixture/normalize}
promptkit:
profiles:
- id: partial
model: fixture-model
api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED
api_key_required: true
prompts:
- id: inspect-initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: inspect-correction
default_profile: partial
messages: [{role: user, content: Correct.}]
output: {format: text, validation_mode: none}
- id: inspect-validator
default_profile: partial
messages: [{role: user, content: Assess.}]
output: {format: text, validation_mode: none}
`
type inspectionArtifact struct {
Value string `json:"value"`
}
type inspectionCodec struct{}
func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" }
func (inspectionCodec) MediaType() string { return "application/json" }
func (inspectionCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)}
}
func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) }
func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) {
return codec.Encode(value)
}
func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) {
return codec.Decode(content)
}
func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) {
var value inspectionArtifact
err := json.Unmarshal(content, &value)
return value, err
}
func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog {
t.Helper()
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec {
return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
}
check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil }))
check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil }))
check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil }))
check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{}))
extract := spec(pipeline.StageExtract, "fixture/extract")
extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"}
extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}}
check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) {
t.Fatal("inspection constructed extractor")
return nil, nil
}))
merge := spec(pipeline.StageMerge, "fixture/merge")
merge.ArtifactKind = "test/inspection"
check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) {
t.Fatal("inspection constructed merger")
return nil, nil
}))
normalize := spec(pipeline.StageNormalize, "fixture/normalize")
normalize.ArtifactKind = "test/inspection"
check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) {
t.Fatal("inspection constructed normalizer")
return nil, nil
}))
check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) {
t.Fatal("inspection constructed validator")
return nil, nil
}))
return catalog
}
type inspectionProvider struct{ t *testing.T }
func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
provider.t.Fatal("inspection called provider")
return nil, nil
}
func writeInspectionConfiguration(t *testing.T, content string) string {
t.Helper()
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
return file
}
func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) {
t.Helper()
var last *nativeLoaders
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders {
last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}}
return last
}, catalog))
if err != nil {
t.Fatal(err)
}
return app, &last
}
func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) {
content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1)
content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1)
file := writeInspectionConfiguration(t, content)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list constructed a catalog")
return pipeline.ModuleCatalog{}, nil
})
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var ids []string
if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || (*state).selections[0].engine != nil {
t.Fatal("list expanded native resources")
}
}
func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var report pipelineDescription
if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 {
t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err)
}
if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 {
t.Fatalf("missing resolved composition: %#v", report)
}
if (*state).selections[0].engine != nil {
t.Fatal("show expanded profiles by constructing engine")
}
assertInspectionNoRunStorage(t, file)
}
func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) {
for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, class)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(out.String(), "demo") {
t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) {
t.Fatal("validation loaded twice or built an unnecessary engine")
}
assertInspectionNoRunStorage(t, file)
}
}
func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) {
for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} {
file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1))
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") {
t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String())
}
}
}
func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
for _, test := range []struct {
args []string
status int
}{
{[]string{"pipeline", "show"}, 2},
{[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2},
{[]string{"pipeline", "show", "unknown", "--config", file}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2},
{[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2},
{[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0},
{[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0},
{[]string{"pipeline", "--help"}, 0},
} {
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != test.status {
t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String())
}
if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 {
t.Fatal("quiet pipeline inspection wrote stdout")
}
}
}
type inspectionFailWriter struct{ err error }
func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err }
func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list loaded catalog")
return pipeline.ModuleCatalog{}, nil
})
primary := errors.New("inspection stdout delivery failed")
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) {
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
}
}
func assertInspectionNoRunStorage(t *testing.T, file string) {
t.Helper()
entries, err := os.ReadDir(filepath.Dir(file))
if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" {
t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err)
}
}