93 lines
3.8 KiB
Go
93 lines
3.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type capturingChunkMapOutput struct{ requests []contracts.OutputRequest }
|
|
|
|
func (*capturingChunkMapOutput) Key() string { return "capture/chunk-map" }
|
|
func (output *capturingChunkMapOutput) Encode(_ context.Context, request contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
output.requests = append(output.requests, request)
|
|
return contracts.OutputResult{}, nil
|
|
}
|
|
|
|
func TestRunnerProvidesAcceptedChunkMapToOutput(t *testing.T) {
|
|
calls, assessments := 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return planResponse(t, typedTestPlan(typedTestDocument())), nil
|
|
}))
|
|
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
|
assessments++
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
})
|
|
_, store := filesystemProducerCache(t)
|
|
encoder := &capturingChunkMapOutput{}
|
|
input.Prepared.output = encoder
|
|
input.RawInput = []byte("input")
|
|
input.Cache = store
|
|
for range 2 {
|
|
if _, err := New().Run(context.Background(), input); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if calls != 1 || assessments != 1 || len(encoder.requests) != 2 {
|
|
t.Fatalf("calls=%d assessments=%d output requests=%d", calls, assessments, len(encoder.requests))
|
|
}
|
|
for _, request := range encoder.requests {
|
|
artifact := request.ChunkMap
|
|
if artifact == nil || artifact.Kind != chunkmap.ArtifactKind || artifact.Schema.ID != chunkmap.SchemaID || artifact.MediaType != chunkmap.MediaType {
|
|
t.Fatalf("chunk map artifact=%#v", artifact)
|
|
}
|
|
value, err := chunkmap.New().Decode(artifact.Content)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantProducer := chunkmap.Producer{InputModule: input.Prepared.input.Key(), ChunkModule: "cached-plan"}
|
|
if value.RequestedChunker != "cached-plan" || value.Producer != wantProducer || len(value.Chunks) != 1 {
|
|
t.Fatalf("chunk map=%#v", value)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerDoesNotPublishRejectedChunkPlan(t *testing.T) {
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return planResponse(t, typedTestPlan(typedTestDocument())), nil
|
|
}))
|
|
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
|
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Choose a valid partition."}, nil
|
|
})
|
|
input.Prepared.resolved.ChunkControls.MaxAdditionalTurns = 0
|
|
encoder := &capturingChunkMapOutput{}
|
|
input.Prepared.output = encoder
|
|
input.RawInput = []byte("input")
|
|
_, err := New().Run(context.Background(), input)
|
|
if err == nil || len(encoder.requests) != 0 {
|
|
t.Fatalf("rejected plan error=%v output requests=%d", err, len(encoder.requests))
|
|
}
|
|
}
|
|
|
|
func TestDeterministicChunkMapDoesNotInventModelProvenance(t *testing.T) {
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
|
|
prepared.resolved.Chunk.LLMProfile = "configured-but-not-used"
|
|
encoder := &capturingChunkMapOutput{}
|
|
prepared.output = encoder
|
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
|
|
t.Fatal("missing chunk map")
|
|
}
|
|
value, err := chunkmap.New().Decode(encoder.requests[0].ChunkMap.Content)
|
|
if err != nil || value.Producer.LLMProfile != "" {
|
|
t.Fatalf("model provenance=%#v %v", value.Producer, err)
|
|
}
|
|
}
|