460 lines
24 KiB
Go
460 lines
24 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
units "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
|
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type runProbe struct {
|
|
mu sync.Mutex
|
|
requests []promptkit.GenerateRequest
|
|
prepared []promptkit.PreparedRun
|
|
references []contracts.ReferenceItem
|
|
engines []*promptkit.Engine
|
|
parsed int
|
|
produced int
|
|
correctFirst bool
|
|
}
|
|
|
|
func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
probe.mu.Lock()
|
|
defer probe.mu.Unlock()
|
|
probe.requests = append(probe.requests, request)
|
|
value := "accepted"
|
|
if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." {
|
|
probe.produced++
|
|
if probe.correctFirst && probe.produced == 1 {
|
|
value = "rejected"
|
|
}
|
|
}
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%q}`, value)}, nil
|
|
}
|
|
func (probe *runProbe) onPrepared(_ context.Context, details promptkit.PreparedRun) {
|
|
probe.mu.Lock()
|
|
defer probe.mu.Unlock()
|
|
probe.prepared = append(probe.prepared, details)
|
|
}
|
|
|
|
type runInputAdapter struct{ probe *runProbe }
|
|
|
|
func (*runInputAdapter) Key() string { return "fixture/input" }
|
|
func (adapter *runInputAdapter) Parse(_ context.Context, request contracts.ParseRequest) (*source.SourceDocument, error) {
|
|
adapter.probe.mu.Lock()
|
|
adapter.probe.parsed++
|
|
adapter.probe.mu.Unlock()
|
|
document := &source.SourceDocument{ID: "source", Kind: "fixture", Format: "text", Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(request.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}}
|
|
var err error
|
|
document.Digest, err = source.DigestDocument(document)
|
|
return document, err
|
|
}
|
|
|
|
type runProducer struct {
|
|
key string
|
|
engine *promptkit.Engine
|
|
probe *runProbe
|
|
}
|
|
|
|
func (producer *runProducer) Key() string { return producer.key }
|
|
func (*runProducer) ReferenceSlots() []contracts.ReferenceSlot {
|
|
return runReferenceSlots()
|
|
}
|
|
func runReferenceSlots() []contracts.ReferenceSlot {
|
|
return []contracts.ReferenceSlot{
|
|
{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain", "application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{"test/inspection"}},
|
|
{Name: "secondary", AcceptedMediaTypes: []string{"text/plain"}},
|
|
}
|
|
}
|
|
func (producer *runProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[inspectionArtifact]) (contracts.TypedExtractionResult[inspectionArtifact], error) {
|
|
content := string(request.SourceInput.Content)
|
|
for _, slot := range request.References.Slots {
|
|
for _, item := range slot.Items {
|
|
content += "\n" + string(item.Content)
|
|
producer.probe.mu.Lock()
|
|
producer.probe.references = append(producer.probe.references, item)
|
|
producer.probe.mu.Unlock()
|
|
}
|
|
}
|
|
initial := promptkit.RunRequest{PromptID: "inspect-initial", ProfileID: request.LLMProfile, SessionID: request.SessionID, Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline(content)}, OnPrepared: producer.probe.onPrepared}
|
|
value, outcome, err := pipeline.RunProducer(ctx, producer.engine, request.Operation, initial, func(_ context.Context, result *promptkit.RunResult) (*inspectionArtifact, []string, error) {
|
|
var value inspectionArtifact
|
|
err := json.Unmarshal([]byte(result.RawOutput), &value)
|
|
return &value, nil, err
|
|
}, func(context.Context, []string) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil
|
|
}, nil)
|
|
return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err
|
|
}
|
|
func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) {
|
|
return contracts.TypedMergeResult[inspectionArtifact]{Value: request.ExtractOutputs[0].Value}, nil
|
|
}
|
|
func (producer *runProducer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[inspectionArtifact]) (contracts.TypedNormalizeResult[inspectionArtifact], error) {
|
|
return contracts.TypedNormalizeResult[inspectionArtifact]{Value: request.MergeOutput.Value}, nil
|
|
}
|
|
|
|
type runValidator struct {
|
|
engine *promptkit.Engine
|
|
probe *runProbe
|
|
}
|
|
|
|
func (*runValidator) Name() string { return "fixture/validator" }
|
|
func (*runValidator) ExecutionClass() contracts.ExecutionClass {
|
|
return contracts.ExecutionClassLLMBacked
|
|
}
|
|
func (validator *runValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[inspectionArtifact]) (contracts.ValidationResult, error) {
|
|
result, err := validator.engine.Run(ctx, promptkit.RunRequest{PromptID: "inspect-validator", ProfileID: request.LLMProfile, SessionID: request.SessionID, Execution: request.Execution.Execution, TransientRetries: request.Execution.TransientRetries, Observer: request.Execution.Observer, OnPrepared: validator.probe.onPrepared})
|
|
if err != nil {
|
|
return contracts.ValidationResult{}, err
|
|
}
|
|
if !result.Validation.IsValid {
|
|
return contracts.ValidationResult{}, fmt.Errorf("native validator contract failed")
|
|
}
|
|
if request.Value.Value != "accepted" {
|
|
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Return accepted."}, nil
|
|
}
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
}
|
|
|
|
func runRegistries(t *testing.T, probe *runProbe) pipeline.Registries {
|
|
t.Helper()
|
|
registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
|
check := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
check(registries.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fixture/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.InputAdapter, error) { return &runInputAdapter{probe}, nil }))
|
|
check(units.Register(registries.Chunkers))
|
|
check(jsonoutput.Register(registries.Outputs))
|
|
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, inspectionCodec{}))
|
|
slots := runReferenceSlots()
|
|
check(pipeline.RegisterExtractorBuilder(registries.Extractors, pipeline.ModuleSpec{Key: "fixture/extract", Stage: pipeline.StageExtract, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"inspect-initial", "inspect-correction"}, ReferenceSlots: slots}, func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }, func(request pipeline.BuildRequest) (contracts.Extractor[inspectionArtifact], error) {
|
|
probe.engines = append(probe.engines, request.Dependencies.Engine)
|
|
return &runProducer{key: "fixture/extract", engine: request.Dependencies.Engine, probe: probe}, nil
|
|
}))
|
|
check(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "fixture/merge", Stage: pipeline.StageMerge, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.Merger[inspectionArtifact], error) {
|
|
return &runProducer{key: "fixture/merge", probe: probe}, nil
|
|
}))
|
|
check(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "fixture/normalize", Stage: pipeline.StageNormalize, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"normalized"}}, func() (contracts.Normalizer[inspectionArtifact], error) {
|
|
return &runProducer{key: "fixture/normalize", probe: probe}, nil
|
|
}))
|
|
check(pipeline.RegisterTypedValidatorBuilder(registries.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"inspect-validator"}}, func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }, func(request pipeline.BuildRequest) (contracts.TypedValidator[inspectionArtifact], error) {
|
|
probe.engines = append(probe.engines, request.Dependencies.Engine)
|
|
return &runValidator{request.Dependencies.Engine, probe}, nil
|
|
}))
|
|
return registries
|
|
}
|
|
|
|
func runFixture(t *testing.T, probe *runProbe, configure ...func(string) string) (*commandkit.App, string, string, **nativeLoaders) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
configuration := strings.ReplaceAll(inspectionConfiguration, "fixture/chunk", "generic")
|
|
configuration = strings.ReplaceAll(configuration, "fixture/output", "json")
|
|
configuration = strings.Replace(configuration, " api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED\n api_key_required: true", " endpoint: https://example.invalid/v1\n api_key_required: false", 1)
|
|
configuration = strings.Replace(configuration, " prompts:", ` - id: chosen
|
|
endpoint: https://example.invalid/v1
|
|
model: chosen-model
|
|
reasoning_effort: high
|
|
temperature: 0.8
|
|
api_key_required: false
|
|
prompts:`, 1)
|
|
for _, update := range configure {
|
|
configuration = update(configuration)
|
|
}
|
|
file := filepath.Join(root, "config.yml")
|
|
input := filepath.Join(root, "input.txt")
|
|
for path, content := range map[string]string{file: configuration, input: "source transcript", filepath.Join(root, "reference-must-not-exist.txt"): "configured reference"} {
|
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
registries := runRegistries(t, probe)
|
|
options := Options{Now: func() time.Time { return time.Unix(42, 0) }}
|
|
runs := 0
|
|
options.RunIDGenerator = func(time.Time) (string, error) { runs++; return fmt.Sprintf("run-%d", runs), nil }
|
|
var state *nativeLoaders
|
|
newState := func() *nativeLoaders {
|
|
state = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(probe)}}
|
|
return state
|
|
}
|
|
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return catalogFromRegistries(registries), nil }, pipelineRunCommand("development", newState, func() (pipeline.Registries, error) { return registries, nil }, options)), commandkit.Alias("run", "pipeline run"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return app, file, input, &state
|
|
}
|
|
|
|
func executeRunFixture(t *testing.T, app *commandkit.App, args ...string) commandkit.Outcome {
|
|
t.Helper()
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 0 {
|
|
writeConfigLoadError(&diagnostics, outcome.Cause)
|
|
t.Fatalf("%v: %#v stderr=%s", args, outcome, diagnostics.String())
|
|
}
|
|
return outcome
|
|
}
|
|
|
|
func TestNativeRunRoutesShareOneHandlerAndApplyOverridesToEveryRole(t *testing.T) {
|
|
probe := &runProbe{correctFirst: true}
|
|
app, file, input, state := runFixture(t, probe)
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-profile", "chosen", "--llm-model", "operator-model", "--llm-session-id", "operator-session", "--llm-reasoning-effort=", "--llm-temperature=0", "--llm-transient-retries=0")
|
|
if probe.produced != 2 || len(probe.requests) != 4 || probe.parsed != 1 {
|
|
t.Fatalf("native operation/correction/validation = %d requests=%d parse=%d", probe.produced, len(probe.requests), probe.parsed)
|
|
}
|
|
for _, request := range probe.requests {
|
|
if request.Target.Model != "operator-model" || request.Prompt.SessionID != "operator-session" || request.Target.ReasoningEffort != "" || request.Target.Temperature != 0 || !request.TargetPresence.Temperature {
|
|
t.Fatalf("override missing on a native role: %#v", request)
|
|
}
|
|
}
|
|
for _, prepared := range probe.prepared {
|
|
if prepared.SelectedProfileID != "chosen" {
|
|
t.Fatalf("profile override missing: %s", prepared.SelectedProfileID)
|
|
}
|
|
}
|
|
if len((*state).selections) != 1 || len(probe.engines) != 2 || probe.engines[0] == nil || probe.engines[0] != probe.engines[1] || probe.engines[0] != (*state).selections[0].engine {
|
|
t.Fatal("run used a second load or engine")
|
|
}
|
|
if (*state).selections[0].result.Config.Pipelines["demo"].LLMProfile != nil {
|
|
t.Fatal("request override mutated captured configuration")
|
|
}
|
|
executeRunFixture(t, app, "pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass")
|
|
for _, id := range []string{"run-1", "run-2"} {
|
|
content, err := os.ReadFile(filepath.Join(filepath.Dir(file), "output-must-not-exist", id, "lanes", "notes.json"))
|
|
if err != nil || !strings.Contains(string(content), "accepted") {
|
|
t.Fatalf("published artifact %s: %s %v", id, content, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeRunSessionIdentityIgnoresReferenceChangesAndHonorsEmptyOverride(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
first := filepath.Join(t.TempDir(), "reference one.txt")
|
|
second := filepath.Join(t.TempDir(), "reference two.txt")
|
|
for path, text := range map[string]string{first: "first reference", second: "second reference"} {
|
|
if err := os.WriteFile(path, []byte(text), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--reference", "context="+first, "--cache", "bypass")
|
|
session := probe.requests[0].Prompt.SessionID
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--reference", "context="+second, "--cache", "bypass")
|
|
if session == "" || session != probe.requests[2].Prompt.SessionID || !strings.Contains(probe.requests[0].Prompt.Messages[0].Content, "first reference") || !strings.Contains(probe.requests[2].Prompt.Messages[0].Content, "second reference") {
|
|
t.Fatal("reference selection changed generated session or failed to reach request")
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/llm_profile="chosen"`, "--llm-session-id=", "--llm-profile=")
|
|
if probe.requests[4].Prompt.SessionID != "" || probe.prepared[4].SelectedProfileID != "partial" {
|
|
t.Fatal("explicit empty native selections were replaced by application defaults")
|
|
}
|
|
}
|
|
|
|
func TestNativeRunValidatesEverySuppliedValueBeforeWork(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
for _, args := range [][]string{
|
|
{"run", "demo", "--config", file},
|
|
{"run", "--config", file, "--input-file", input},
|
|
{"run", "demo", "--config", file, "--input-file", "", "--input-file", input},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--cache", "bad", "--cache", "auto"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--output-dir", "", "--output-dir", t.TempDir()},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--reference", "invalid"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--without-reference", "context"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--llm-temperature=invalid", "--llm-temperature=0"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--only", "notes"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--json"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--resume"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--recompute-step", "build"},
|
|
{"run", "demo", "--config", file, "--input-file", input, "--chunk-cache", "auto"},
|
|
} {
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 2 {
|
|
t.Fatalf("%v: %#v %s", args, outcome, diagnostics.String())
|
|
}
|
|
}
|
|
if len(probe.requests) != 0 || probe.parsed != 0 {
|
|
t.Fatal("invalid invocation started source/model execution")
|
|
}
|
|
}
|
|
|
|
func TestNativeRunDebugAndOutputOverridesRemainInvocationRelative(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
output := t.TempDir()
|
|
debug := t.TempDir()
|
|
workingDirectory, err := os.Getwd()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
relativeOutput, err := filepath.Rel(workingDirectory, output)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
relativeDebug, err := filepath.Rel(workingDirectory, debug)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--output-dir", relativeOutput, "--debug", "--debug-dir", relativeDebug, "--cache", "bypass")
|
|
if _, err := os.Stat(filepath.Join(output, "run-1", "lanes", "notes.json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(debug, "run-1", "outcome.json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--debug", "--debug-dir=", "--cache", "bypass")
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "debug-must-not-exist")); !os.IsNotExist(err) {
|
|
t.Fatalf("explicit empty debug path allocated capture: %v", err)
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--debug", "--debug=false", "--debug-dir", debug, "--cache", "bypass")
|
|
if _, err := os.Stat(filepath.Join(debug, "run-3")); !os.IsNotExist(err) {
|
|
t.Fatalf("explicit false debug selection allocated capture: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunReferencesKeepSpecificityAndFileOrigins(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe, func(configuration string) string {
|
|
return strings.Replace(configuration, "references: {context: {path: reference-must-not-exist.txt}}", "references: {context: {path: reference-must-not-exist.txt}, secondary: {path: never-read.txt}}", 1)
|
|
})
|
|
workingDirectory, err := os.Getwd()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths := make(map[string]string)
|
|
for _, name := range []string{"broad", "lane", "binding", "secondary"} {
|
|
absolute := filepath.Join(t.TempDir(), name+" reference.txt")
|
|
if err := os.WriteFile(absolute, []byte(name+" reference"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths[name], err = filepath.Rel(workingDirectory, absolute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass",
|
|
"--reference", "notes.extract.context="+paths["binding"],
|
|
"--reference", "notes.context="+paths["lane"],
|
|
"--reference", "context="+paths["broad"],
|
|
"--reference", "secondary="+paths["secondary"])
|
|
if len(probe.references) != 2 {
|
|
t.Fatalf("materialized references: %#v", probe.references)
|
|
}
|
|
for _, item := range probe.references {
|
|
name := "binding"
|
|
if string(item.Content) == "secondary reference" {
|
|
name = "secondary"
|
|
}
|
|
absolute := filepath.Clean(filepath.Join(workingDirectory, paths[name]))
|
|
if string(item.Content) != name+" reference" || item.BindingSource != contracts.ReferenceBindingSourceCLI || item.Origin.URI != (&url.URL{Scheme: "file", Path: absolute}).String() {
|
|
t.Fatalf("CLI reference content/origin: %#v", item)
|
|
}
|
|
}
|
|
probe.references = nil
|
|
executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass",
|
|
"--without-reference", "secondary")
|
|
configured := (&url.URL{Scheme: "file", Path: filepath.Join(filepath.Dir(file), "reference-must-not-exist.txt")}).String()
|
|
if len(probe.references) != 1 || string(probe.references[0].Content) != "configured reference" || probe.references[0].BindingSource != contracts.ReferenceBindingSourceConfig || probe.references[0].Origin.URI != configured {
|
|
t.Fatalf("config reference origin or optional unbind: %#v", probe.references)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunLiteralLanesAndFinalSingletonValues(t *testing.T) {
|
|
probe := &runProbe{}
|
|
lane := `{"extract":{"module":"fixture/extract","references":{"context":{"path":"reference-must-not-exist.txt"}},"validators":[{"module":"fixture/validator"}]},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}`
|
|
artifacts := `{"notes,extra":` + lane + `,"other":` + lane + `,"unselected":` + lane + `}`
|
|
app, file, input, _ := runFixture(t, probe, func(configuration string) string {
|
|
start := strings.Index(configuration, " steps:")
|
|
end := strings.Index(configuration, "promptkit:")
|
|
return configuration[:start] + " artifacts: " + artifacts + "\n" + configuration[end:]
|
|
})
|
|
output := t.TempDir()
|
|
executeRunFixture(t, app, "run", "--config", file,
|
|
"--input-file", filepath.Join(t.TempDir(), "earlier-input.txt"), "--input-file", input,
|
|
"--output-dir", filepath.Join(t.TempDir(), "earlier-output"), "--output-dir", output,
|
|
"--llm-profile", "not-the-final-profile", "--llm-profile", "chosen",
|
|
"--cache", "auto", "--cache", "bypass",
|
|
"--lane", "notes,extra", "--lane", "other", "--", "demo")
|
|
if probe.produced != 2 || probe.parsed != 1 {
|
|
t.Fatalf("selected lanes or singleton input: produced=%d parsed=%d", probe.produced, probe.parsed)
|
|
}
|
|
files, err := os.ReadDir(filepath.Join(output, "run-1", "lanes"))
|
|
if err != nil || len(files) != 2 {
|
|
t.Fatalf("selected output files: %#v %v", files, err)
|
|
}
|
|
for _, prepared := range probe.prepared {
|
|
if prepared.SelectedProfileID != "chosen" {
|
|
t.Fatalf("final singleton profile not used: %q", prepared.SelectedProfileID)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "cache-must-not-exist")); !os.IsNotExist(err) {
|
|
t.Fatalf("final bypass value allocated cache: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunProtectsGeneratedReferencesBeforeReadingFiles(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, _ := runFixture(t, probe)
|
|
steps := `[{"id":"produce","artifacts":{"earlier":{"extract":{"module":"fixture/extract","references":{"context":{"path":"reference-must-not-exist.txt"}}},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}}},{"id":"consume","artifacts":{"later":{"extract":{"module":"fixture/extract","references":{"context":{"artifact":{"step":"produce","lane":"earlier"}}}},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}}}]`
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/pipelines/demo/steps=" + steps, "--reference", "context=never-read.txt"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 2 || !strings.Contains(diagnostics.String(), "conflicting generated and external bindings") || len(probe.requests) != 0 || probe.parsed != 0 {
|
|
t.Fatalf("generated/external conflict: %#v %s requests=%d parsed=%d", outcome, diagnostics.String(), len(probe.requests), probe.parsed)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunCapturesPreflightFailureBeforeSourceExecution(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, _, state := runFixture(t, probe)
|
|
content, err := os.ReadFile(file)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content = bytes.Replace(content, []byte("id: inspect-correction"), []byte("id: different-correction"), 1)
|
|
if err := os.WriteFile(file, content, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing-input.txt"), "--debug"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 2 || !strings.Contains(diagnostics.String(), "inspect-correction") || probe.parsed != 0 || len(probe.requests) != 0 || len(probe.engines) != 0 {
|
|
t.Fatalf("preflight order: %#v %s", outcome, diagnostics.String())
|
|
}
|
|
if (*state).selections[0].engine == nil {
|
|
t.Fatal("preflight did not inspect the captured engine")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1", "outcome.json")); err != nil {
|
|
t.Fatalf("failed preflight capture: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNativeRunDebugAllocationFailurePreventsEngineConstruction(t *testing.T) {
|
|
probe := &runProbe{}
|
|
app, file, input, state := runFixture(t, probe)
|
|
debug := filepath.Join(t.TempDir(), "not-a-directory")
|
|
if err := os.WriteFile(debug, []byte("occupied"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out, diagnostics bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--debug", "--debug-dir", debug}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
if outcome.Status != 1 || probe.parsed != 0 || len(probe.requests) != 0 || len(probe.engines) != 0 || (*state).selections[0].engine != nil {
|
|
t.Fatalf("capture allocation order: %#v %s", outcome, diagnostics.String())
|
|
}
|
|
}
|