277 lines
13 KiB
Go
277 lines
13 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
const inspectionConfiguration = `version: 6
|
|
output: {directory: output-must-not-exist}
|
|
cache_directory: cache-must-not-exist
|
|
debug: {directory: debug-must-not-exist}
|
|
pipelines:
|
|
demo:
|
|
input: {module: fixture/input}
|
|
chunk: {module: fixture/chunk}
|
|
output: {module: fixture/output}
|
|
steps:
|
|
- id: build
|
|
artifacts:
|
|
notes:
|
|
extract:
|
|
module: fixture/extract
|
|
references: {context: {path: reference-must-not-exist.txt}}
|
|
validators: [{module: fixture/validator}]
|
|
merge: {module: fixture/merge}
|
|
normalize: {module: fixture/normalize}
|
|
promptkit:
|
|
profiles:
|
|
- id: partial
|
|
model: fixture-model
|
|
api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED
|
|
api_key_required: true
|
|
prompts:
|
|
- id: inspect-initial
|
|
default_profile: partial
|
|
inputs: [{name: source, required: true}]
|
|
messages: [{role: user, content: '{{input "source"}}'}]
|
|
output: {format: text, validation_mode: none}
|
|
- id: inspect-correction
|
|
default_profile: partial
|
|
messages: [{role: user, content: Correct.}]
|
|
output: {format: text, validation_mode: none}
|
|
- id: inspect-validator
|
|
default_profile: partial
|
|
messages: [{role: user, content: Assess.}]
|
|
output: {format: text, validation_mode: none}
|
|
`
|
|
|
|
type inspectionArtifact struct {
|
|
Value string `json:"value"`
|
|
}
|
|
type inspectionCodec struct{}
|
|
|
|
func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" }
|
|
func (inspectionCodec) MediaType() string { return "application/json" }
|
|
func (inspectionCodec) Schema() contracts.ArtifactSchema {
|
|
return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)}
|
|
}
|
|
func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) }
|
|
func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) {
|
|
return codec.Encode(value)
|
|
}
|
|
func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) {
|
|
return codec.Decode(content)
|
|
}
|
|
func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) {
|
|
var value inspectionArtifact
|
|
err := json.Unmarshal(content, &value)
|
|
return value, err
|
|
}
|
|
|
|
func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog {
|
|
t.Helper()
|
|
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
|
check := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
|
|
}
|
|
check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil }))
|
|
check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil }))
|
|
check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil }))
|
|
check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{}))
|
|
extract := spec(pipeline.StageExtract, "fixture/extract")
|
|
extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"}
|
|
extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}}
|
|
check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) {
|
|
t.Fatal("inspection constructed extractor")
|
|
return nil, nil
|
|
}))
|
|
merge := spec(pipeline.StageMerge, "fixture/merge")
|
|
merge.ArtifactKind = "test/inspection"
|
|
check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) {
|
|
t.Fatal("inspection constructed merger")
|
|
return nil, nil
|
|
}))
|
|
normalize := spec(pipeline.StageNormalize, "fixture/normalize")
|
|
normalize.ArtifactKind = "test/inspection"
|
|
check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) {
|
|
t.Fatal("inspection constructed normalizer")
|
|
return nil, nil
|
|
}))
|
|
check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) {
|
|
t.Fatal("inspection constructed validator")
|
|
return nil, nil
|
|
}))
|
|
return catalog
|
|
}
|
|
|
|
type inspectionProvider struct{ t *testing.T }
|
|
|
|
func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
provider.t.Fatal("inspection called provider")
|
|
return nil, nil
|
|
}
|
|
|
|
func writeInspectionConfiguration(t *testing.T, content string) string {
|
|
t.Helper()
|
|
file := filepath.Join(t.TempDir(), "config.yml")
|
|
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return file
|
|
}
|
|
|
|
func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) {
|
|
t.Helper()
|
|
var last *nativeLoaders
|
|
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders {
|
|
last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}}
|
|
return last
|
|
}, catalog))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return app, &last
|
|
}
|
|
|
|
func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) {
|
|
content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1)
|
|
content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1)
|
|
file := writeInspectionConfiguration(t, content)
|
|
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
|
|
t.Fatal("list constructed a catalog")
|
|
return pipeline.ModuleCatalog{}, nil
|
|
})
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
var ids []string
|
|
if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
|
|
t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String())
|
|
}
|
|
if len((*state).selections) != 1 || (*state).selections[0].engine != nil {
|
|
t.Fatal("list expanded native resources")
|
|
}
|
|
}
|
|
|
|
func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) {
|
|
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
|
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
|
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
var report pipelineDescription
|
|
if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 {
|
|
t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err)
|
|
}
|
|
if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 {
|
|
t.Fatalf("missing resolved composition: %#v", report)
|
|
}
|
|
if (*state).selections[0].engine != nil {
|
|
t.Fatal("show expanded profiles by constructing engine")
|
|
}
|
|
assertInspectionNoRunStorage(t, file)
|
|
}
|
|
|
|
func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) {
|
|
for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} {
|
|
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
|
catalog := inspectionCatalog(t, class)
|
|
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 0 || !strings.Contains(out.String(), "demo") {
|
|
t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String())
|
|
}
|
|
if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) {
|
|
t.Fatal("validation loaded twice or built an unnecessary engine")
|
|
}
|
|
assertInspectionNoRunStorage(t, file)
|
|
}
|
|
}
|
|
|
|
func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) {
|
|
for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} {
|
|
file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1))
|
|
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
|
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") {
|
|
t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) {
|
|
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
|
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
|
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
|
for _, test := range []struct {
|
|
args []string
|
|
status int
|
|
}{
|
|
{[]string{"pipeline", "show"}, 2},
|
|
{[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2},
|
|
{[]string{"pipeline", "show", "unknown", "--config", file}, 2},
|
|
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2},
|
|
{[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2},
|
|
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2},
|
|
{[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0},
|
|
{[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0},
|
|
{[]string{"pipeline", "--help"}, 0},
|
|
} {
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != test.status {
|
|
t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String())
|
|
}
|
|
if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 {
|
|
t.Fatal("quiet pipeline inspection wrote stdout")
|
|
}
|
|
}
|
|
}
|
|
|
|
type inspectionFailWriter struct{ err error }
|
|
|
|
func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err }
|
|
|
|
func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
|
|
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
|
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
|
|
t.Fatal("list loaded catalog")
|
|
return pipeline.ModuleCatalog{}, nil
|
|
})
|
|
primary := errors.New("inspection stdout delivery failed")
|
|
var diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
|
|
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
|
|
}
|
|
}
|
|
|
|
func assertInspectionNoRunStorage(t *testing.T, file string) {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(filepath.Dir(file))
|
|
if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" {
|
|
t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err)
|
|
}
|
|
}
|