Compare commits
65 Commits
9790579b51
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 63e26dc203 | |||
| fe44b12073 | |||
| 65b8e8f0a2 | |||
| 76364a7b7e | |||
| 1c93aa0947 | |||
| 1febd2edfa | |||
| 5e269f9760 | |||
| fb58aef8c2 | |||
| 3f9727d5e8 | |||
| 0663fe566a | |||
| 347ef05b41 | |||
| 068a4b1b5a | |||
| b396581d21 | |||
| 6f0a75e7cf | |||
| d88a01cec0 | |||
| c9744ad2e6 | |||
| 74e0f454e9 | |||
| 244f258413 | |||
| 5bab35f096 | |||
| 8136b4814f | |||
| 349ace0528 | |||
| 716bc62991 | |||
| 61ceab786c | |||
| 2d1fa57d9b | |||
| 6da4d070e5 | |||
| ae0e410875 | |||
| 79088a3546 | |||
| 2c8b63187c | |||
| 2b278c4074 | |||
| db32d31708 | |||
| b9899ea3cc | |||
| 66fa5585c1 | |||
| e690cae24f | |||
| 6700ed2188 | |||
| 4aba4f5672 | |||
| 169a94f7ea | |||
| db8acbea43 | |||
| 79154882b8 | |||
| c848befb53 | |||
| a904e4d841 | |||
| 7ea9fe12b9 | |||
| deaa3560bc | |||
| ef504eabb2 | |||
| ec2ac3369b | |||
| 8a8ca6d83c | |||
| 30509f9d33 | |||
| 28aab5e9a4 | |||
| ea403c8094 | |||
| f5b066beee | |||
| aae01a59ce | |||
| 71cf084e8b | |||
| e327a5eef7 | |||
| af62b3bacc | |||
| e1097cc7a5 | |||
| 3f226c91d0 | |||
| 64509947df | |||
| 29c7c165a5 | |||
| 4ef69314e1 | |||
| 1d1434b118 | |||
| f4779fede9 | |||
| 01c08ec8e4 | |||
| d7ea0fd32d | |||
| ec99d48619 | |||
| 9e74df0b6e | |||
| 6e5e277426 |
@@ -4,6 +4,6 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: validate-release
|
- name: validate-release
|
||||||
image: golang:1.25.5
|
image: golang:1.27.0
|
||||||
commands:
|
commands:
|
||||||
- ./scripts/check-release-source.sh "$CI_COMMIT_TAG"
|
- ./scripts/check-release-source.sh "$CI_COMMIT_TAG"
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ demonstrates all implemented D&D lanes and the supporting campaign references.
|
|||||||
|
|
||||||
## Install A Source Release
|
## Install A Source Release
|
||||||
|
|
||||||
|
Source builds require Go at or above the consumer minimum declared in
|
||||||
|
[`go.mod`](go.mod). Go 1.27.0 is required by the selected ConfigKit
|
||||||
|
dependency.
|
||||||
|
|
||||||
Install a pinned source release with Go:
|
Install a pinned source release with Go:
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
|
|||||||
@@ -1,45 +1,46 @@
|
|||||||
id: dnd.combat_turns
|
prompts:
|
||||||
version: "v1"
|
dnd.combat_turns:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: npc_registry
|
content_type: text/plain
|
||||||
required: false
|
- name: npc_registry
|
||||||
content_type: application/json
|
required: false
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_combat_turns_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_combat_turns_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,53 +1,54 @@
|
|||||||
id: dnd.enemy_events
|
prompts:
|
||||||
version: "v1"
|
dnd.enemy_events:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: npc_registry
|
content_type: text/plain
|
||||||
required: true
|
- name: npc_registry
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: combat_turns
|
content_type: application/json
|
||||||
required: true
|
- name: combat_turns
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: npc_occurrences
|
content_type: application/json
|
||||||
required: true
|
- name: npc_occurrences
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||||
content_file: ./combat-grounding.md
|
- role: user
|
||||||
- role: user
|
content_file: ./combat-grounding.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_enemy_events_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_enemy_events_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,45 +1,46 @@
|
|||||||
id: dnd.item_occurrences
|
prompts:
|
||||||
version: "v1"
|
dnd.item_occurrences:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: item_registry
|
content_type: text/plain
|
||||||
required: true
|
- name: item_registry
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./item-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./item-registry.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_item_occurrences_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_item_occurrences_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,40 +1,41 @@
|
|||||||
id: dnd.item_registry
|
prompts:
|
||||||
version: "v1"
|
dnd.item_registry:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
messages:
|
content_type: text/plain
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_item_registry_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_item_registry_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,30 +1,31 @@
|
|||||||
id: dnd.item_registry.normalize
|
prompts:
|
||||||
version: "v1"
|
dnd.item_registry.normalize:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: candidates
|
inputs:
|
||||||
required: true
|
- name: candidates
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: transcript
|
content_type: application/json
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/protocol.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/protocol.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/candidates.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/candidates.md
|
||||||
content_file: ./sharedassets/transcript-windows.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,45 +1,46 @@
|
|||||||
id: dnd.location_occurrences
|
prompts:
|
||||||
version: "v1"
|
dnd.location_occurrences:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: location_registry
|
content_type: text/plain
|
||||||
required: true
|
- name: location_registry
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./location-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./location-registry.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_location_occurrences_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_location_occurrences_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,40 +1,41 @@
|
|||||||
id: dnd.location_registry
|
prompts:
|
||||||
version: "v1"
|
dnd.location_registry:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
messages:
|
content_type: text/plain
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_location_registry_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_location_registry_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,30 +1,31 @@
|
|||||||
id: dnd.location_registry.normalize
|
prompts:
|
||||||
version: "v1"
|
dnd.location_registry.normalize:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: candidates
|
inputs:
|
||||||
required: true
|
- name: candidates
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: transcript
|
content_type: application/json
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/protocol.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/protocol.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/candidates.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/candidates.md
|
||||||
content_file: ./sharedassets/transcript-windows.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,45 +1,46 @@
|
|||||||
id: dnd.npc_occurrences
|
prompts:
|
||||||
version: "v1"
|
dnd.npc_occurrences:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: npc_registry
|
content_type: text/plain
|
||||||
required: true
|
- name: npc_registry
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_npc_occurrences_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_npc_occurrences_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,40 +1,41 @@
|
|||||||
id: dnd.npc_registry
|
prompts:
|
||||||
version: "v1"
|
dnd.npc_registry:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
messages:
|
content_type: text/plain
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_npc_registry_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_npc_registry_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,30 +1,31 @@
|
|||||||
id: dnd.npc_registry.normalize
|
prompts:
|
||||||
version: "v1"
|
dnd.npc_registry.normalize:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: candidates
|
inputs:
|
||||||
required: true
|
- name: candidates
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: transcript
|
content_type: application/json
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/protocol.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/protocol.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/candidates.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/candidates.md
|
||||||
content_file: ./sharedassets/transcript-windows.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/transcript-windows.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
id: dnd-extraction
|
profiles:
|
||||||
backend: openrouter
|
dnd-extraction:
|
||||||
model: openai/gpt-5.6-luna
|
backend: openrouter
|
||||||
timeout_seconds: 240
|
model: openai/gpt-5.6-luna
|
||||||
service_tier: flex
|
timeout_seconds: 240
|
||||||
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,40 +1,41 @@
|
|||||||
id: dnd.scene_descriptions
|
prompts:
|
||||||
version: "v1"
|
dnd.scene_descriptions:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
messages:
|
content_type: text/plain
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_scene_descriptions_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_scene_descriptions_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,23 +1,24 @@
|
|||||||
id: dnd.scene_descriptions.validate_combat
|
prompts:
|
||||||
version: "v1"
|
dnd.scene_descriptions.validate_combat:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-scene-combat-policy.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
- role: user
|
content_file: ./instructions.md
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_scene_combat_semantics_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,34 +1,35 @@
|
|||||||
id: dnd.scenes
|
prompts:
|
||||||
version: "v1"
|
dnd.scenes:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
messages:
|
content_type: text/plain
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
- role: user
|
content_file: ./instructions.md
|
||||||
content_file: ./sharedassets/common-dnd-transcript-full.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-full.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_scenes_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_scenes_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,50 +1,51 @@
|
|||||||
id: dnd.spells
|
prompts:
|
||||||
version: "v1"
|
dnd.spells:
|
||||||
default_profile: dnd-extraction
|
v1:
|
||||||
inputs:
|
default_profile: dnd-extraction
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: spell_catalog
|
content_type: application/json
|
||||||
required: true
|
- name: spell_catalog
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: players
|
content_type: application/json
|
||||||
required: false
|
- name: players
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: party
|
content_type: text/plain
|
||||||
required: false
|
- name: party
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: glossary
|
content_type: text/plain
|
||||||
required: false
|
- name: glossary
|
||||||
content_type: text/plain
|
required: false
|
||||||
- name: npc_registry
|
content_type: text/plain
|
||||||
required: false
|
- name: npc_registry
|
||||||
content_type: application/json
|
required: false
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./sharedassets/common-dnd-system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
content_file: ./sharedassets/common-dnd-identity.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
content_file: ./sharedassets/common-dnd-references.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./sharedassets/common-dnd-transcript-chunk.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
content_file: ./sharedassets/common-dnd-npc-registry.md
|
- role: user
|
||||||
- role: user
|
content_file: ./sharedassets/common-dnd-npc-registry.md
|
||||||
content_file: ./spell-catalog.md
|
- role: user
|
||||||
- role: user
|
content_file: ./spell-catalog.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
output:
|
type: ephemeral
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: dnd_spells_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: dnd_spells_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,27 +1,28 @@
|
|||||||
id: generic.semantic_reconciliation
|
prompts:
|
||||||
version: "v1"
|
generic.semantic_reconciliation:
|
||||||
inputs:
|
v1:
|
||||||
- name: candidates
|
inputs:
|
||||||
required: true
|
- name: candidates
|
||||||
content_type: application/json
|
required: true
|
||||||
- name: transcript
|
content_type: application/json
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: system
|
messages:
|
||||||
content_file: ./system.md
|
- role: system
|
||||||
- role: user
|
content_file: ./system.md
|
||||||
content_file: ./protocol.md
|
- role: user
|
||||||
- role: user
|
content_file: ./protocol.md
|
||||||
content_file: ./instructions.md
|
- role: user
|
||||||
cache_control:
|
content_file: ./instructions.md
|
||||||
type: ephemeral
|
cache_control:
|
||||||
- role: user
|
type: ephemeral
|
||||||
content_file: ./candidates.md
|
- role: user
|
||||||
- role: user
|
content_file: ./candidates.md
|
||||||
content_file: ./transcript-windows.md
|
- role: user
|
||||||
output:
|
content_file: ./transcript-windows.md
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: semantic_reconciliation_llm.v1.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 1
|
schema_path: semantic_reconciliation_llm.v1.json
|
||||||
|
repair_attempts: 1
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# ADR-0010: Use workload-oriented LLM profile defaults
|
# ADR-0010: Use workload-oriented LLM profile defaults
|
||||||
|
|
||||||
**Status:** Accepted
|
**Status:** Accepted; profile source replacement behavior superseded by
|
||||||
|
[ADR-0019](0019-adopt-native-promptkit-configuration.md)
|
||||||
**Date:** 2026-08-03
|
**Date:** 2026-08-03
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
|
|||||||
92
docs/adr/0019-adopt-native-promptkit-configuration.md
Normal file
92
docs/adr/0019-adopt-native-promptkit-configuration.md
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
# ADR-0019: Adopt native PromptKit configuration sources
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-10-04
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
PromptKit v0.12 replaces its former source options with ConfigKit-backed
|
||||||
|
configuration sources, keyed documents, recursive overlays, and native
|
||||||
|
inheritance. Notarius must choose how application-owned prompt and fallback
|
||||||
|
assets, operator configuration, and the optional local backend reach PromptKit
|
||||||
|
without recreating source parsing or inheritance in the adapter. Runtime,
|
||||||
|
preflight, and operator inspection must agree about the effective definitions.
|
||||||
|
|
||||||
|
Notarius also captures checkpoint identity for the PromptKit configuration
|
||||||
|
used by an execution engine. That identity must follow the engine's captured
|
||||||
|
definition state rather than re-reading mutable source files, and must not
|
||||||
|
publish credentials, prompt bodies, provider parameters, or source paths.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Use PromptKit's public v0.12 native source APIs as the only PromptKit
|
||||||
|
configuration boundary. Assemble PromptKit catalogs, registered application
|
||||||
|
prompt and fallback inputs, the optional operator file or recursively selected
|
||||||
|
directory, and the optional Notarius local backend registration in that order.
|
||||||
|
Use native ConfigKit imports and PromptKit/ConfigKit composition behavior;
|
||||||
|
retain each filesystem input's resource context. Inputs within a group compose
|
||||||
|
strictly, and later groups recursively overlay earlier ones. PromptKit owns
|
||||||
|
definition parsing, exact-key `extends` inheritance, and final effective-value
|
||||||
|
validation. Notarius does not provide a legacy parser, compatibility alias,
|
||||||
|
filename-priority merger, whole-profile replacement shim, or alternate
|
||||||
|
inheritance engine.
|
||||||
|
|
||||||
|
Execution, explicit-profile preflight, and configuration inspection use the
|
||||||
|
same source construction. Each constructed engine captures its effective
|
||||||
|
definitions. Edits to source files take effect for a newly constructed engine,
|
||||||
|
not an existing one. Derive the private versioned
|
||||||
|
`promptkit_configuration` checkpoint identity from public captured inspection
|
||||||
|
and definition APIs, including effective configuration, profile targets and
|
||||||
|
credential-policy metadata, captured prompt hashes, and the PromptKit and
|
||||||
|
selected catalog identities. Retain only the digest as checkpoint metadata;
|
||||||
|
keep all digest inputs private. Exclude local-backend concurrency settings
|
||||||
|
because they affect scheduling rather than execution semantics.
|
||||||
|
|
||||||
|
Keep PromptKit types and source assembly inside the LLM integration boundary.
|
||||||
|
The CLI may project copied profile fields into a Notarius-owned neutral JSON
|
||||||
|
contract. PromptKit's effective and explanation reports remain upstream
|
||||||
|
redacted configuration and source-history views. Neither inspection path reads
|
||||||
|
credential values or contacts a provider.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The migration to v0.12 keyed configuration is an intentional external file
|
||||||
|
format change. Operators must migrate `id`/`version` embedded fields and
|
||||||
|
`base_profile` to native keyed documents and `extends`. Notarius does not
|
||||||
|
silently accept the former option-based API or file format. Notarius's own
|
||||||
|
version 5 configuration contract and loader decision in
|
||||||
|
[ADR-0018](0018-adopt-configkit.md) remain in force; this decision concerns
|
||||||
|
PromptKit definitions and source composition.
|
||||||
|
|
||||||
|
The workload-oriented profile identity and D&D fallback ownership in
|
||||||
|
[ADR-0010](0010-workload-oriented-llm-profile-defaults.md) remain in force.
|
||||||
|
Only that ADR's former claim that a matching operator profile completely
|
||||||
|
replaces the fallback has been superseded: native recursive overlays retain
|
||||||
|
unspecified inherited fields.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Keep the removed PromptKit option-based configuration API or add a
|
||||||
|
compatibility parser. This would fork the supported upstream contract and
|
||||||
|
preserve legacy format machinery.
|
||||||
|
- Merge profile documents inside Notarius or restore whole-profile
|
||||||
|
replacement. This would duplicate upstream composition behavior and make
|
||||||
|
inspection, preflight, and runtime easier to diverge.
|
||||||
|
- Re-read and hash source files for checkpoint identity. This could describe
|
||||||
|
mutable files rather than the definitions captured by the engine and would
|
||||||
|
duplicate upstream discovery and composition.
|
||||||
|
- Expose upstream inspection structs directly as CLI output. This would bind
|
||||||
|
the public Notarius contract to dependency-owned Go types.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
PromptKit remains the owner of its native configuration semantics, while
|
||||||
|
Notarius owns which registered and operator sources are assembled and when an
|
||||||
|
engine is reconstructed. A valid later overlay can supersede an earlier
|
||||||
|
definition; validation applies to the final effective definitions rather than
|
||||||
|
as a separate audit of every superseded value. Configuration reports can
|
||||||
|
contain ordinary prompt and provider parameters, so operators own the
|
||||||
|
destination and retention of explicitly requested reports even though
|
||||||
|
credential-shaped fields are redacted. PromptKit upgrades require review of
|
||||||
|
the source adapter, captured identity, reports, and the pinned upstream
|
||||||
|
contracts.
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# ADR-0020: Delegate configuration mechanics to ConfigKit
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
**Date:** 2026-10-04
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
[ADR-0018](0018-adopt-configkit.md) establishes ConfigKit as Notarius's
|
||||||
|
configuration loader. This decision clarifies the continuing ownership boundary
|
||||||
|
as ConfigKit's capabilities evolve.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
ConfigKit owns configuration mechanics: source loading, parsing, imports,
|
||||||
|
composition, overlays, inheritance, presence tracking, provenance, and
|
||||||
|
inspection. Notarius adopts those semantics through ConfigKit's public APIs.
|
||||||
|
|
||||||
|
Notarius owns application meaning: its configuration schema, version, defaults,
|
||||||
|
field normalization, validity checks, and pipeline/module compatibility. It
|
||||||
|
supplies application-specific defaults, normalization, and validation through
|
||||||
|
ConfigKit's supported extension points. Source-origin metadata supports
|
||||||
|
application rules such as declaration-relative filesystem paths.
|
||||||
|
|
||||||
|
Notarius selects configuration inputs and options through the public API.
|
||||||
|
ConfigKit resolves those inputs and delivers the final application document
|
||||||
|
with source composition and application-field normalization complete. Notarius
|
||||||
|
converts that document into its runtime configuration and performs
|
||||||
|
catalog-dependent pipeline resolution and runtime preparation afterward.
|
||||||
|
|
||||||
|
Operator-requested effective configuration and setting explanations use
|
||||||
|
ConfigKit's inspection APIs. Application validation evaluates the resolved
|
||||||
|
configuration. Changes to general configuration mechanics are pursued upstream
|
||||||
|
in ConfigKit and adopted through its supported API.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- Application-owned configuration loading provides direct control over source
|
||||||
|
semantics and assigns parsing, composition, and provenance maintenance to
|
||||||
|
Notarius.
|
||||||
|
- Shared ownership of configuration mechanics allows local adaptations and
|
||||||
|
requires maintaining coordinated semantics across ConfigKit and Notarius.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
ConfigKit provides a coherent configuration-resolution and inspection contract.
|
||||||
|
Notarius's configuration work concentrates on application semantics and operator
|
||||||
|
integration. Upstream upgrades are evaluated against this ownership boundary
|
||||||
|
and the application's deliberately selected requirements. This decision
|
||||||
|
clarifies ADR-0018 while retaining its adoption and application-format decisions.
|
||||||
70
docs/cli.md
70
docs/cli.md
@@ -6,6 +6,13 @@ Configuration fields, discovery rules, and selectable module keys are defined
|
|||||||
in [Configuration](config.md); runtime state and recovery procedures are
|
in [Configuration](config.md); runtime state and recovery procedures are
|
||||||
defined in [Operations](operations.md).
|
defined in [Operations](operations.md).
|
||||||
|
|
||||||
|
Configuration-consuming commands share the same selection: explicit `--config`,
|
||||||
|
then `NOTARIUS_CONFIG`, then working-directory, user, and system searches. A
|
||||||
|
selected file's loading failure is terminal. Reports use that selected source;
|
||||||
|
the [configuration reference](config.md#configuration-discovery-and-precedence)
|
||||||
|
defines the filename groups and value precedence. Use `--config` for a fixed
|
||||||
|
service or subprocess selection.
|
||||||
|
|
||||||
## Command Summary
|
## Command Summary
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
@@ -13,6 +20,9 @@ notarius help
|
|||||||
notarius --version
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
|
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
|
||||||
|
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
@@ -188,8 +198,16 @@ notarius config effective [--config path/to/config.yml] [--config-overlay name]
|
|||||||
~~~
|
~~~
|
||||||
|
|
||||||
This command writes the selected composed configuration as deterministic,
|
This command writes the selected composed configuration as deterministic,
|
||||||
redacted YAML. Standard output contains only that YAML: it does not validate a
|
redacted YAML. By default, it reports the Notarius configuration. Set
|
||||||
pipeline, prepare modules, materialize references, or contact a provider.
|
**--scope promptkit** to report PromptKit's captured effective configuration;
|
||||||
|
**--scope notarius** is equivalent to the default. Only the PromptKit scope
|
||||||
|
constructs the PromptKit inspection engine, using the selected configuration
|
||||||
|
and registered application assets. Engine construction validates and captures
|
||||||
|
the definitions. Standard output contains only the selected report: the
|
||||||
|
command does not validate a pipeline, prepare modules, materialize references,
|
||||||
|
resolve credentials, or contact a provider. PromptKit output can include
|
||||||
|
ordinary inline prompt and provider-parameter data; credential-shaped provider
|
||||||
|
parameters are redacted.
|
||||||
|
|
||||||
## config explain
|
## config explain
|
||||||
|
|
||||||
@@ -197,11 +215,49 @@ pipeline, prepare modules, materialize references, or contact a provider.
|
|||||||
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
This command writes ConfigKit's redacted, human-readable explanation for one
|
This command writes a redacted, human-readable explanation for one canonical
|
||||||
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
|
JSON Pointer. Supply `""` explicitly to explain the root. By default, it
|
||||||
pointer is a syntax error; a well-formed unknown path is a command failure.
|
explains Notarius configuration; **--scope promptkit** selects PromptKit
|
||||||
The explanation can describe a value displaced by a selected overlay. Standard
|
configuration history, while **--scope notarius** is equivalent to the default.
|
||||||
output contains only the explanation.
|
A malformed pointer is a syntax error; a well-formed unknown path is a command
|
||||||
|
failure. The explanation can describe values displaced by source overlays or
|
||||||
|
imports. Standard output contains only the selected explanation. PromptKit
|
||||||
|
scope constructs and validates its captured definitions, and its report can
|
||||||
|
include ordinary inline prompt and provider-parameter data.
|
||||||
|
|
||||||
|
## config profile
|
||||||
|
|
||||||
|
~~~
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
This command inspects one exact, nonblank PromptKit profile ID and writes one
|
||||||
|
JSON object followed by a newline:
|
||||||
|
|
||||||
|
The following illustrates the JSON shape; field values vary with the selected
|
||||||
|
profile and its configured backend.
|
||||||
|
|
||||||
|
~~~json
|
||||||
|
{"profile_id":"example","api_key_required":true,"effective_model_params":{"backend_id":"openrouter","endpoint":"https://provider.example/v1","model":"example-model","temperature":0,"max_tokens":0,"top_p":0,"timeout_seconds":600,"service_tier":"","reasoning_effort":"","api_key_env":"PROVIDER_API_KEY","extra_params":null}}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
The object reports the resolved target, including backend-completed endpoint
|
||||||
|
and credential-name settings. An empty **backend_id** identifies an
|
||||||
|
endpoint-only profile. Credential names and **api_key_required** are configured
|
||||||
|
metadata; the command does not look up credential values or report whether an
|
||||||
|
environment variable is set. Credential-shaped keys in **extra_params** are
|
||||||
|
redacted.
|
||||||
|
|
||||||
|
This is the named profile's configured target, not a prediction of provider
|
||||||
|
defaults or the profile selected by an individual request. A request may omit
|
||||||
|
its profile and let its prompt select a default, and a run may supply a profile
|
||||||
|
override; this command applies neither behavior. Zero-valued fields show the
|
||||||
|
effective value but do not distinguish omission from an explicit zero. Use
|
||||||
|
[effective configuration](#config-effective) and
|
||||||
|
[configuration explanation](#config-explain) to inspect source presence and
|
||||||
|
history. Profile/configuration failures use exit status 1; malformed command
|
||||||
|
syntax, a blank ID, or an incorrect argument count use exit status 2. No
|
||||||
|
transcript or model call is involved.
|
||||||
|
|
||||||
## pipelines list
|
## pipelines list
|
||||||
|
|
||||||
|
|||||||
299
docs/config.md
299
docs/config.md
@@ -1,7 +1,7 @@
|
|||||||
# Configuration
|
# Configuration
|
||||||
|
|
||||||
This is the canonical reference for Notarius configuration. Configuration files
|
This is the canonical reference for Notarius configuration. Configuration files
|
||||||
are YAML and must declare version 5. They select pipelines and their modules;
|
are YAML; the main file must declare version 5. They select pipelines and their modules;
|
||||||
the [CLI reference](cli.md) owns invocation syntax, and
|
the [CLI reference](cli.md) owns invocation syntax, and
|
||||||
[Operations](operations.md) owns run-state procedures.
|
[Operations](operations.md) owns run-state procedures.
|
||||||
|
|
||||||
@@ -11,17 +11,30 @@ Commands that load configuration choose a file in this order:
|
|||||||
|
|
||||||
1. a non-empty **--config** CLI value;
|
1. a non-empty **--config** CLI value;
|
||||||
2. a non-empty **NOTARIUS_CONFIG** environment value;
|
2. a non-empty **NOTARIUS_CONFIG** environment value;
|
||||||
3. the installed default file at **/usr/local/etc/notarius/config.yml**, when
|
3. `notarius.yml`, then `notarius.yaml` in the working directory;
|
||||||
it exists.
|
4. the user configuration directory; then
|
||||||
|
5. the Unix system configuration directories.
|
||||||
|
|
||||||
The command fails if none of these paths provides a configuration file.
|
The user search checks `<user-config>/notarius/` for `notarius.yml`,
|
||||||
|
`notarius.yaml`, `config.yml`, `config.yaml`, `main.yml`, and `main.yaml`, in
|
||||||
|
that order, then `<user-config>/notarius.yml` and `notarius.yaml`. The system
|
||||||
|
search uses the same groups under `/usr/local/etc/notarius/` and `/etc/notarius/`,
|
||||||
|
then `/etc/notarius.yml` and `/etc/notarius.yaml`. Generic filenames in the
|
||||||
|
working directory are not candidates. Home-directory search is not enabled.
|
||||||
|
|
||||||
|
Discovery selects one main file; it does not layer all candidates. A missing
|
||||||
|
candidate advances the search, but an explicitly selected missing file, access
|
||||||
|
error, or invalid selected file fails without fallback. An exhausted search
|
||||||
|
fails. Use an explicit `--config` for services and subprocesses that need a
|
||||||
|
fixed selection independent of their working directory.
|
||||||
|
|
||||||
For configuration values, precedence is:
|
For configuration values, precedence is:
|
||||||
|
|
||||||
1. built-in defaults;
|
1. built-in defaults;
|
||||||
2. composed YAML sources, including imports;
|
2. composed YAML sources, including imports;
|
||||||
3. the selected ConfigKit overlay; and
|
3. the selected ConfigKit overlay and inheritance;
|
||||||
4. explicitly supplied configuration-valued run flags.
|
4. typed decoding and explicitly supplied configuration-valued run flags; and
|
||||||
|
5. Notarius resolution and validation.
|
||||||
|
|
||||||
`--config-overlay` selects a ConfigKit overlay from `overlays/<name>.yml`
|
`--config-overlay` selects a ConfigKit overlay from `overlays/<name>.yml`
|
||||||
beneath the selected configuration root. Empty and `default` select no overlay.
|
beneath the selected configuration root. Empty and `default` select no overlay.
|
||||||
@@ -38,7 +51,23 @@ have no effect; `NOTARIUS_CONFIG` remains a source-discovery input.
|
|||||||
ordered steps, all implemented D&D lanes, generated references, state
|
ordered steps, all implemented D&D lanes, generated references, state
|
||||||
settings, bounded LLM concurrency, and the maintained
|
settings, bounded LLM concurrency, and the maintained
|
||||||
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
|
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) and imports its
|
||||||
pipeline from [a fragment](../examples/pipelines/dnd-session.yml).
|
pipeline from [a fragment](../examples/pipelines/dnd-session.yml), which imports
|
||||||
|
its [first step](../examples/pipelines/dnd-session.describe-session.yml).
|
||||||
|
|
||||||
|
The complete fragment also defines `dnd-session-development`, which inherits
|
||||||
|
the complete pipeline and profile while setting
|
||||||
|
`structured_output_repair_attempts: 0`. Select it with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
notarius run dnd-session-development \
|
||||||
|
--config /path/to/dnd-complete.config.yml \
|
||||||
|
--input /path/to/transcript.json
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [CLI reference](cli.md#run) for the full invocation contract. This
|
||||||
|
variant disables PromptKit structural repairs without changing configured producer
|
||||||
|
retry budgets. When using another profile, operators can instead select
|
||||||
|
`dnd-session` and set its `llm_profile` to that profile.
|
||||||
|
|
||||||
Use these complete files as starting points rather than combining the
|
Use these complete files as starting points rather than combining the
|
||||||
illustrative fragments in this reference.
|
illustrative fragments in this reference.
|
||||||
@@ -80,16 +109,118 @@ umask, setgid inheritance, and default ACLs; see [Operations](operations.md#cach
|
|||||||
|
|
||||||
Every nonempty relative configured destination or reference path resolves from
|
Every nonempty relative configured destination or reference path resolves from
|
||||||
the canonical directory of the file that supplies its surviving declaration.
|
the canonical directory of the file that supplies its surviving declaration.
|
||||||
This includes imported fragments and selected configuration overlays. Built-in
|
This includes imported fragments, selected configuration overlays, and values
|
||||||
relative destinations use the selected main configuration file's directory;
|
copied through inheritance: the copied value keeps its original declaring
|
||||||
explicit `--output-dir` and `--debug-dir` values use the invocation working
|
file. A child-supplied replacement resolves from the child's declaring file.
|
||||||
directory. Absolute values remain unchanged.
|
Built-in relative destinations use the selected main configuration file's
|
||||||
|
directory; explicit `--output-dir` and `--debug-dir` values use the invocation
|
||||||
|
working directory. Absolute values remain unchanged.
|
||||||
|
|
||||||
The root file can use ConfigKit imports to compose disjoint definitions. Imports
|
Use source-root `config` controls for imports and collection declarations. For
|
||||||
and selected overlays remain under the configuration root; they do not merge or
|
example, an essential fragment should explicitly be required:
|
||||||
append a pipeline's `steps` sequence. An overlay replaces sequences as defined
|
|
||||||
by ConfigKit, so changing an individual ordered binding requires supplying the
|
~~~yaml
|
||||||
whole replacement sequence.
|
config:
|
||||||
|
imports:
|
||||||
|
- path: pipelines/session.yml
|
||||||
|
required: true
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Imports are optional by default. ConfigKit supports paths, lists, directories,
|
||||||
|
globs, exclusions, recursive selection, destinations, and sequence-item placement;
|
||||||
|
its native input-root rules apply. Imported fragments need not declare a version.
|
||||||
|
The complete example places a mapping fragment at a step-list destination. Nested
|
||||||
|
application data named `config` is ordinary data, not a control block.
|
||||||
|
|
||||||
|
When migrating application files, rename genuine root `configkit` control
|
||||||
|
blocks to `config` in the main file and fragments. This is not a compatibility
|
||||||
|
alias: `configkit` is ordinary data under the new reader. PromptKit profile
|
||||||
|
documents still use their separate older engine and retain `configkit` controls;
|
||||||
|
see [PromptKit Profiles](#promptkit-profiles).
|
||||||
|
|
||||||
|
## Sibling Inheritance
|
||||||
|
|
||||||
|
ConfigKit supports native `extends` composition in mappings and identified
|
||||||
|
sequence records. In a mapping, `extends: production` selects the named immediate
|
||||||
|
sibling in that same mapping. In an identified sequence, the reference is an
|
||||||
|
identity mapping: for steps identified by `id`, `extends: {id: parent}` selects
|
||||||
|
a record in the same final sequence. Neither form is a path or cross-scope
|
||||||
|
lookup. See the pinned [native inheritance examples](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md#sibling-inheritance).
|
||||||
|
Imports and the selected overlay are composed before
|
||||||
|
inheritance and typed decoding. Explicit CLI assignments follow decoding and
|
||||||
|
inheritance, before Notarius resolution and validation; they cannot repair an
|
||||||
|
earlier structural or decoding error. Every definition, including a parent and definitions not
|
||||||
|
selected for a run, remains subject to ordinary validation.
|
||||||
|
|
||||||
|
Child mappings recursively overlay parent mappings. Scalars and null replace
|
||||||
|
inherited values. Unidentified sequences replace as whole values; pipeline
|
||||||
|
steps are identified by `id` and use native upsert by default:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
pipelines:
|
||||||
|
production:
|
||||||
|
input: {module: seriatim}
|
||||||
|
llm_profile: dnd-extraction
|
||||||
|
# Other complete pipeline settings and ordered steps...
|
||||||
|
development:
|
||||||
|
extends: production
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Pipeline names are siblings within `pipelines`; nested mappings such as named
|
||||||
|
artifact lanes also have their own sibling namespace. Distinct imported step IDs
|
||||||
|
combine in ConfigKit's dependency-ready assembly order, not necessarily import
|
||||||
|
declaration or acquisition order; duplicate peer IDs conflict. Ready contributions
|
||||||
|
are selected by their stable source keys. See the pinned
|
||||||
|
[import assembly contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md#explicit-imports)
|
||||||
|
for full ordering semantics. In the complete example, the imported describe-session
|
||||||
|
step precedes the importing body's extract-events step because an imported
|
||||||
|
dependency is assembled before its importer. Across an overlay or
|
||||||
|
inheritance, upsert replaces a matching step as a complete record in place and
|
||||||
|
appends new IDs. It does not merge individual fields of a matching step. An
|
||||||
|
identified record can itself use native `extends`. Authors may select `replace`
|
||||||
|
or `append` with a `config.sequences` declaration at the relevant destination;
|
||||||
|
an explicit replacement with `[]` clears the list, while empty append or upsert
|
||||||
|
does not. Notarius then validates the resulting pipeline, including its nonempty
|
||||||
|
step requirement. Validator chains are not automatically identified. Authors
|
||||||
|
may declare identities for other lists when needed. CLI list assignment replaces
|
||||||
|
the complete decoded list.
|
||||||
|
|
||||||
|
Record inheritance does not create template-only steps: every record in the
|
||||||
|
final step list executes and must have artifact lanes. A child that inherits
|
||||||
|
its parent's named lanes also conflicts with the requirement that lane IDs be
|
||||||
|
unique across steps; adding differently named lanes does not remove inherited
|
||||||
|
ones. Use pipeline-level inheritance with step upsert for pipeline variants.
|
||||||
|
Native identified-record inheritance is also available in compatible collections,
|
||||||
|
such as an author-identified validator list, subject to ordinary domain validation.
|
||||||
|
|
||||||
|
For example, an overlay can opt into replacing a pipeline's steps:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
config:
|
||||||
|
sequences:
|
||||||
|
- destination: [pipelines, production, steps]
|
||||||
|
mode: replace
|
||||||
|
pipelines:
|
||||||
|
production:
|
||||||
|
steps:
|
||||||
|
- id: extract
|
||||||
|
artifacts: {spells: {extract: {module: dnd/spells}}}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Inherited values count as explicit input for Notarius's existing
|
||||||
|
presence-sensitive rules: zero, false, and empty-list overrides stay explicit
|
||||||
|
instead of restoring a default. Null also replaces the inherited value, but
|
||||||
|
each field's ordinary type and validation rules still apply; null is not
|
||||||
|
universally valid. Omitting `extends` preserves a relationship supplied by a
|
||||||
|
lower-priority source, while `extends: null` cancels that relationship. It
|
||||||
|
does not independently clear other inherited fields.
|
||||||
|
|
||||||
|
`extends` is now a reserved key in every input mapping, including dynamic
|
||||||
|
module options and named maps. Existing ordinary data or entry names using that
|
||||||
|
key must be renamed along with any consumer that reads them. Configuration
|
||||||
|
overlays, pipeline inheritance, and PromptKit's native profile `extends`
|
||||||
|
inheritance are separate mechanisms.
|
||||||
|
|
||||||
## PromptKit Profiles
|
## PromptKit Profiles
|
||||||
|
|
||||||
@@ -124,14 +255,16 @@ file that supplies it. The complete example's
|
|||||||
Use absolute paths for services and containers when their deployment layout is
|
Use absolute paths for services and containers when their deployment layout is
|
||||||
not configuration-relative.
|
not configuration-relative.
|
||||||
|
|
||||||
An operator source is optional. For a requested ID, PromptKit checks the
|
An operator source is optional. PromptKit's maintained catalog, Notarius's
|
||||||
configured operator source first, then Notarius's embedded fallback profiles,
|
registered prompt and fallback assets, and the selected operator source are
|
||||||
then its own built-in catalog. A matching profile is complete: it replaces a
|
assembled in that order; a configured `local_backend` contributes its backend
|
||||||
lower-precedence definition rather than merging with it. The maintained
|
last. Later source groups recursively overlay earlier groups. Within one group,
|
||||||
|
inputs compose strictly; directory filenames do not provide priority. For a
|
||||||
|
requested profile ID, an operator definition can partially override the
|
||||||
|
embedded fallback while retaining unspecified fields. The maintained
|
||||||
[`dnd-extraction` LLM profile](../examples/llm-profiles/dnd-extraction.yml)
|
[`dnd-extraction` LLM profile](../examples/llm-profiles/dnd-extraction.yml)
|
||||||
is a secret-free deployment artifact; production, development, and local
|
is a secret-free deployment artifact. It also demonstrates a derived
|
||||||
deployments can each provide a complete definition with that same workload ID.
|
development profile that is not selected by the normal pipeline.
|
||||||
Use workload-oriented IDs for new profiles instead of model names.
|
|
||||||
[Operations](operations.md#promptkit-profile-deployment) owns the deployment
|
[Operations](operations.md#promptkit-profile-deployment) owns the deployment
|
||||||
workflow and credential-handling guidance.
|
workflow and credential-handling guidance.
|
||||||
|
|
||||||
@@ -144,27 +277,125 @@ a positive value limits simultaneous local generations. The application-wide
|
|||||||
backend field has an environment override. Omitting **local_backend** registers
|
backend field has an environment override. Omitting **local_backend** registers
|
||||||
nothing and preserves existing built-in and endpoint-only profile behavior.
|
nothing and preserves existing built-in and endpoint-only profile behavior.
|
||||||
|
|
||||||
A file-backed PromptKit profile selects the registration by its case-sensitive
|
A PromptKit file uses keyed mappings for profiles, backends, and prompts. A
|
||||||
backend ID:
|
profile selects a backend by its exact ID:
|
||||||
|
|
||||||
~~~yaml
|
~~~yaml
|
||||||
id: local-summary
|
profiles:
|
||||||
backend: local
|
local-summary:
|
||||||
model: example-model
|
backend: local
|
||||||
|
model: example-model
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||||
its credential environment variable through `api_key_env`; set that variable
|
its credential environment variable through `api_key_env`; set that variable
|
||||||
only in the run environment. PromptKit owns the
|
only in the run environment. `api_key_required` is supported in profile files;
|
||||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md),
|
inspection reports PromptKit's effective metadata but does not check whether
|
||||||
including `base_profile` inheritance. Notarius passes profiles through without
|
the named environment variable is set. PromptKit owns the
|
||||||
merging them. Filesystem profiles cannot express PromptKit's in-memory
|
[pinned v0.12.0 format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md),
|
||||||
`APIKeyRequired` setting; an unset `api_key_env` is optional and may reach the
|
including native `extends` inheritance and field-presence rules. Notarius
|
||||||
provider without authorization.
|
selects source groups and passes definitions through PromptKit.
|
||||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||||
supported package API, and [Operations](operations.md#operational-limits)
|
supported package API, and [Operations](operations.md#operational-limits)
|
||||||
describes the effective concurrency layers.
|
describes the effective concurrency layers.
|
||||||
|
|
||||||
|
### Native PromptKit Sources And Migration
|
||||||
|
|
||||||
|
The configured `profile_file` or `profile_dir` is a PromptKit configuration
|
||||||
|
source. It can contain profiles and backends, as well as prompt overlays.
|
||||||
|
PromptKit's catalog is followed by registered application prompt/fallback
|
||||||
|
assets and the selected operator source; Notarius's optional programmatic
|
||||||
|
`local` backend is applied last. Composition, recursion, exact-key inheritance,
|
||||||
|
and final-value validation are owned by PromptKit and ConfigKit. Explicit
|
||||||
|
`null`, zero, false, blank, and omission may have different meanings; an empty
|
||||||
|
map does not clear an inherited map.
|
||||||
|
|
||||||
|
When migrating external files from PromptKit before v0.12, place backend and
|
||||||
|
profile entries beneath their `backends` and `profiles` mappings, and place
|
||||||
|
prompt versions beneath `prompts.<prompt-id>.v1`. Remove embedded prompt
|
||||||
|
`id`/`version` fields. Replace `base_profile` with native `extends`. There is
|
||||||
|
no legacy-format parser or alias layer. Keep Notarius's `version: 5` and
|
||||||
|
`promptkit.profile_file`/`profile_dir` settings unchanged.
|
||||||
|
|
||||||
|
Imports use native ConfigKit declarations relative to the file that declares
|
||||||
|
them. A file-root layout keeps imported fragments outside a recursively
|
||||||
|
selected profile directory, avoiding selection of the same definition both by
|
||||||
|
directory discovery and by import:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
# notarius.yml
|
||||||
|
version: 5
|
||||||
|
promptkit:
|
||||||
|
profile_file: ./promptkit/entry.yml
|
||||||
|
~~~
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
# promptkit/entry.yml
|
||||||
|
configkit:
|
||||||
|
imports:
|
||||||
|
- path: fragments/profiles.yml
|
||||||
|
profiles:
|
||||||
|
dnd-extraction-development:
|
||||||
|
extends: dnd-extraction
|
||||||
|
temperature: 0.2
|
||||||
|
~~~
|
||||||
|
|
||||||
|
PromptKit recursively discovers YAML files in a selected directory, and its
|
||||||
|
inputs compose strictly; there is no filename-priority merger. Imports retain
|
||||||
|
their declaring source roots, so relative message/content files resolve from
|
||||||
|
the file that declares them. Use a separate file root for related imports
|
||||||
|
when directory selection would otherwise load those files a second time.
|
||||||
|
|
||||||
|
The same unified document can declare a custom backend or select a catalog
|
||||||
|
backend without redeclaring it:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
backends:
|
||||||
|
private-compatible:
|
||||||
|
endpoint: https://llm.example.invalid/v1
|
||||||
|
api_key_env: PRIVATE_LLM_API_KEY
|
||||||
|
profiles:
|
||||||
|
private-summary:
|
||||||
|
backend: private-compatible
|
||||||
|
model: summary-model
|
||||||
|
catalog-summary:
|
||||||
|
backend: openrouter
|
||||||
|
model: openai/gpt-5.6-luna
|
||||||
|
~~~
|
||||||
|
|
||||||
|
To override the embedded D&D fallback for a configured local backend, supply
|
||||||
|
only the changed fields. Explicitly clear inherited settings that do not apply
|
||||||
|
to the local target:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
profiles:
|
||||||
|
dnd-extraction:
|
||||||
|
backend: local
|
||||||
|
model: local-model
|
||||||
|
service_tier: null
|
||||||
|
~~~
|
||||||
|
|
||||||
|
The backend can be declared as `backends.local` in the operator source or
|
||||||
|
registered through the optional `promptkit.local_backend` convenience object.
|
||||||
|
The [source precedence](#native-promptkit-sources-and-migration) applies when
|
||||||
|
both contribute. Prompt overlays use the same recursive source composition; for
|
||||||
|
example, this changes one output setting while retaining the other embedded
|
||||||
|
definition fields:
|
||||||
|
|
||||||
|
~~~yaml
|
||||||
|
prompts:
|
||||||
|
dnd.spells:
|
||||||
|
v1:
|
||||||
|
output:
|
||||||
|
repair_attempts: 0
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Sequences such as prompt messages are replaced as a whole when overridden.
|
||||||
|
Engine construction validates final effective prompt definitions and captures
|
||||||
|
message files and static templates. Request preparation loads response schemas
|
||||||
|
and resolves request variables and input artifacts. Profile preflight needs no
|
||||||
|
transcript, credential value, or provider call, but can read prompt-content files.
|
||||||
|
|
||||||
`notarius config validate --pipeline <id>` resolves the selected pipeline and
|
`notarius config validate --pipeline <id>` resolves the selected pipeline and
|
||||||
inspects every explicit effective profile without contacting a provider or
|
inspects every explicit effective profile without contacting a provider or
|
||||||
requiring credential values. It rejects absent, malformed, or incompatible
|
requiring credential values. It rejects absent, malformed, or incompatible
|
||||||
|
|||||||
@@ -1,18 +1,21 @@
|
|||||||
# ConfigKit Integration
|
# ConfigKit Integration
|
||||||
|
|
||||||
Notarius uses the public root package
|
Notarius application configuration uses
|
||||||
[`gitea.maximumdirect.net/eric/ConfigKit`](https://gitea.maximumdirect.net/eric/ConfigKit)
|
[`go.maximumdirect.net/configkit`](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e)
|
||||||
at v0.4.2 for version 5 configuration acquisition, imports, overlays, explicit
|
at v1.2.0 for source selection, imports, overlays, identified collections,
|
||||||
overrides, diagnostics, and safe inspection rendering.
|
inheritance, explicit overrides, diagnostics, and frozen inspection. Imports
|
||||||
|
and inheritance precede typed decoding; CLI assignments follow decoding and
|
||||||
|
precede application resolution. Notarius does not repeat source composition.
|
||||||
|
|
||||||
Notarius owns the typed document, domain defaults and validation, runtime
|
Notarius supplies the document schema, defaults, normalization, validation,
|
||||||
conversion, catalog-dependent pipeline resolution, source-aware configured
|
and the step identity registration. It owns runtime conversion and subsequent
|
||||||
paths, and command discovery. It uses only ConfigKit's public path, result,
|
catalog-dependent pipeline resolution. `LoadSource` receives the CLI's chosen
|
||||||
diagnostic, and resolution-source APIs. Upstream documentation owns general
|
source; retained main-source evidence supplies the selected path. See
|
||||||
ConfigKit semantics; [Configuration](../config.md) owns Notarius's file and
|
[Configuration](../config.md) for the operator contract and the pinned
|
||||||
operator contract.
|
upstream [consumer guide](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/consumers/README.md)
|
||||||
|
and [configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/docs/config.md).
|
||||||
|
|
||||||
The v0.4.2 patch updates upstream release tooling and policy/licensing
|
PromptKit v0.12.0 separately retains old-path ConfigKit v0.5.0 for its exposed
|
||||||
documentation without changing configuration behavior or the public API. See
|
source and inspection types. The two module identities share YAML v4.0.0-rc.6;
|
||||||
the canonical [ConfigKit attribution](../oss/attribution.md#configkit) for its
|
their named types remain inside their owning boundaries. See the
|
||||||
license and notices.
|
[PromptKit integration](pkg-promptkit.md) and [attribution](../oss/attribution.md#configkit).
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# PromptKit Integration
|
# PromptKit Integration
|
||||||
|
|
||||||
Notarius pins
|
Notarius pins
|
||||||
[`gitea.maximumdirect.net/eric/promptkit` v0.11.2](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2)
|
[`gitea.maximumdirect.net/eric/promptkit` v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0)
|
||||||
as its in-process prompt engine. The upstream
|
as its in-process prompt engine. The upstream
|
||||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
|
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||||
owns the public engine API, and the upstream
|
owns the public engine API, and the upstream
|
||||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md)
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
|
||||||
owns prompt, profile, and schema file contracts.
|
owns prompt, profile, and schema file contracts.
|
||||||
|
|
||||||
## Supported Boundary
|
## Supported Boundary
|
||||||
@@ -20,15 +20,20 @@ Notarius relies on the root `promptkit` package to:
|
|||||||
record credential-redacted details and run that exact execution;
|
record credential-redacted details and run that exact execution;
|
||||||
- return rendered debug material, validated structured output, selected
|
- return rendered debug material, validated structured output, selected
|
||||||
profile, backend, effective model metadata, and token usage;
|
profile, backend, effective model metadata, and token usage;
|
||||||
- register the optional conventional `local` backend through `BackendLocal`,
|
- compose profile and backend mappings through `ConfigurationSources`,
|
||||||
`LocalBackend`, and `WithBackend`;
|
`ProfileSource`, and `BackendSource`, retaining filesystem roots with
|
||||||
|
`FilesystemInputs` and `DirectoryInputs`;
|
||||||
|
- register the optional conventional `local` backend with `BackendSource`,
|
||||||
|
`BackendLocal`, and `LocalBackend`;
|
||||||
- distinguish structured-output validation failure from execution failure; and
|
- distinguish structured-output validation failure from execution failure; and
|
||||||
- identify a missing explicit profile through `ErrProfileNotFound` and backend
|
- identify a missing explicit profile through `ErrProfileNotFound` and backend
|
||||||
admission exhaustion through `ErrCapacityExceeded`.
|
admission exhaustion through `ErrCapacityExceeded`.
|
||||||
|
|
||||||
The pinned
|
The pinned
|
||||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/backends.go)
|
[`BackendSource`, `BackendLocal`, and `LocalBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/backends.go)
|
||||||
owns the registration and backend-capacity contract.
|
owns keyed backend configuration and backend-capacity semantics. The
|
||||||
|
[package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
describes source-group assembly.
|
||||||
|
|
||||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||||
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
|
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
|
||||||
@@ -72,7 +77,7 @@ PromptKit capacity rejection into the provider-neutral Notarius
|
|||||||
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
|
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
|
||||||
backend ID in safe diagnostic context, without exposing PromptKit's capacity
|
backend ID in safe diagnostic context, without exposing PromptKit's capacity
|
||||||
error type. PromptKit's built-in client retries eligible transient HTTP failures
|
error type. PromptKit's built-in client retries eligible transient HTTP failures
|
||||||
with its v0.11.2 default of two additional attempts per generation; Notarius
|
with its v0.12.0 default of two additional attempts per generation; Notarius
|
||||||
keeps pipeline-stage retries outside that completion boundary.
|
keeps pipeline-stage retries outside that completion boundary.
|
||||||
|
|
||||||
The adapter projects built-in generation failures as the application-owned
|
The adapter projects built-in generation failures as the application-owned
|
||||||
@@ -84,20 +89,28 @@ exposing PromptKit error values to Notarius callers.
|
|||||||
## Profile Sources And Compatibility
|
## Profile Sources And Compatibility
|
||||||
|
|
||||||
Notarius gives PromptKit the configured operator profile source, registered
|
Notarius gives PromptKit the configured operator profile source, registered
|
||||||
application fallback profile assets, and optional backend registration through
|
application fallback profile assets, and optional backend configuration
|
||||||
the same construction path for inspection and execution. PromptKit owns the
|
through the same construction path for inspection and execution. PromptKit
|
||||||
resulting source precedence and strict profile parsing: a matching operator
|
owns source precedence, strict group composition, and recursive field overlay:
|
||||||
profile is a complete replacement for a fallback or built-in profile, while an
|
omitted operator fields retain lower-layer values, while supplied fields
|
||||||
invalid matching document fails instead of falling through. The operator
|
replace them. The application group retains registered filesystem roots; the
|
||||||
|
operator may be one file with native imports or a recursively discovered
|
||||||
|
directory. Imports resolve resources from their declaring file. Prompt and
|
||||||
|
profile definitions are validated after source composition, so a malformed
|
||||||
|
lower-layer value superseded by a valid overlay is not separately audited. A
|
||||||
|
malformed final effective definition fails construction. The operator
|
||||||
configuration and deployment workflow are defined in
|
configuration and deployment workflow are defined in
|
||||||
[Configuration](../config.md#promptkit-profiles) and
|
[Configuration](../config.md#promptkit-profiles) and
|
||||||
[Operations](../operations.md#promptkit-profile-deployment).
|
[Operations](../operations.md#promptkit-profile-deployment).
|
||||||
|
|
||||||
PromptKit owns `base_profile` resolution under its
|
PromptKit and ConfigKit own `extends` inheritance under the
|
||||||
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/formats.md).
|
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md).
|
||||||
Notarius records the selected leaf identity and resolved target without parsing
|
Notarius records the selected leaf identity and resolved target without parsing
|
||||||
or merging inheritance. An unset filesystem `api_key_env` is optional and may
|
or merging inheritance. `api_key_env` names the environment variable; inspection
|
||||||
reach the provider without authorization, which can result in a 401 or 403.
|
does not check whether it is set. PromptKit's effective `api_key_required`
|
||||||
|
metadata is supported in file-backed profiles and reported as metadata, not
|
||||||
|
as an availability check. Whether a usable credential reaches the provider
|
||||||
|
remains an execution concern.
|
||||||
|
|
||||||
PromptKit accepts only the `developer`, `system`, `user`, and
|
PromptKit accepts only the `developer`, `system`, `user`, and
|
||||||
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
`assistant` text-chat roles after normalizing case and surrounding whitespace.
|
||||||
@@ -124,16 +137,21 @@ PromptKit obtains its maintained OpenRouter and Rakestrawhome backend and
|
|||||||
profile catalogs from independently versioned transitive modules. Notarius
|
profile catalogs from independently versioned transitive modules. Notarius
|
||||||
does not import or register either catalog; PromptKit retains catalog source,
|
does not import or register either catalog; PromptKit retains catalog source,
|
||||||
identity, precedence, credential, and capacity ownership.
|
identity, precedence, credential, and capacity ownership.
|
||||||
|
The selected integration modules are
|
||||||
|
[`promptkit-backend-openrouter/v3`](../oss/attribution.md#promptkit-openrouter-catalog)
|
||||||
|
and
|
||||||
|
[`promptkit-backend-rakestrawhome/v3`](../oss/attribution.md#promptkit-rakestrawhome-catalog),
|
||||||
|
both pinned at `v3.20260930.0` for this compatibility boundary.
|
||||||
|
|
||||||
External catalog profile names and settings can change between dependency
|
External catalog profile names and settings can change between dependency
|
||||||
releases, including removal without aliases. Review directly selected IDs and
|
releases, including removal without aliases. Review directly selected IDs and
|
||||||
inherited `base_profile` values when upgrading. An application-owned alias that
|
inherited `extends` values when upgrading. An application-owned alias that
|
||||||
inherits an external profile does not freeze that profile's settings.
|
inherits an external profile does not freeze that profile's settings.
|
||||||
PromptKit's opt-in `promptkit-openrouter-standard` and
|
PromptKit's opt-in `promptkit-openrouter-standard` and
|
||||||
`promptkit-openrouter-auto` names provide name continuity within its
|
`promptkit-openrouter-auto` names provide name continuity within its
|
||||||
selected-and-tested dependency scope, not a fixed model or provider availability
|
selected-and-tested dependency scope, not a fixed model or provider availability
|
||||||
guarantee. See the upstream
|
guarantee. See the upstream
|
||||||
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/policy/release.md#catalog-profile-compatibility).
|
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/policy/release.md#catalog-profile-compatibility).
|
||||||
|
|
||||||
Use a complete operator-owned profile with explicit backend or endpoint, model,
|
Use a complete operator-owned profile with explicit backend or endpoint, model,
|
||||||
and settings when those choices must remain operator-controlled. The embedded
|
and settings when those choices must remain operator-controlled. The embedded
|
||||||
@@ -141,13 +159,28 @@ and settings when those choices must remain operator-controlled. The embedded
|
|||||||
external catalog profile. An upgrade updates Notarius's compiled-in catalog
|
external catalog profile. An upgrade updates Notarius's compiled-in catalog
|
||||||
checkpoint identity so prior profile-source checkpoints are not reused.
|
checkpoint identity so prior profile-source checkpoints are not reused.
|
||||||
|
|
||||||
Notarius supports this boundary against PromptKit v0.11.2. Its fallback source,
|
Notarius supports this boundary against PromptKit v0.12.0. Its native sources,
|
||||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||||
profile-source construction, and this compatibility statement against the
|
profile-source construction, and this compatibility statement against the
|
||||||
pinned upstream documentation.
|
pinned upstream documentation.
|
||||||
|
|
||||||
|
PromptKit v0.12.0 exposes old-path ConfigKit v0.5.0 types and reads its operator
|
||||||
|
documents with that engine. Notarius application configuration uses the separate
|
||||||
|
new-path ConfigKit integration; the adapter does not translate source graphs
|
||||||
|
between the two.
|
||||||
|
|
||||||
|
Captured inspection describes one engine's immutable effective view. Source
|
||||||
|
edits require constructing a new engine to become visible. Effective and
|
||||||
|
explanation reports retain PromptKit's redacted configuration and source
|
||||||
|
history contracts; the profile command is a separate Notarius-owned projection
|
||||||
|
of copied, provider-neutral metadata. None of these views reads credential
|
||||||
|
values or predicts provider defaults. See the public [CLI reference](../cli.md#config-effective),
|
||||||
|
[explanation](../cli.md#config-explain), and
|
||||||
|
[profile](../cli.md#config-profile) entries for their syntax and output
|
||||||
|
contracts.
|
||||||
|
|
||||||
## Notarius Ownership
|
## Notarius Ownership
|
||||||
|
|
||||||
[LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts
|
[LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts
|
||||||
@@ -171,8 +204,9 @@ production prompts use PromptKit's bounded structural-repair contract; their
|
|||||||
current declaration is one additional repair attempt. Notarius retains the
|
current declaration is one additional repair attempt. Notarius retains the
|
||||||
transport-neutral boundary and does not expose PromptKit types to modules.
|
transport-neutral boundary and does not expose PromptKit types to modules.
|
||||||
|
|
||||||
The v0.11.2 patch updates upstream policy and licensing documentation without
|
The v0.12.0 upgrade replaces the removed option-based configuration API with
|
||||||
changing runtime APIs, profile settings, or dependency/catalog versions.
|
native ConfigKit-backed sources and keyed documents; release pins and notices
|
||||||
|
are listed in the [canonical attribution](../oss/attribution.md#promptkit).
|
||||||
See the canonical [PromptKit attribution](../oss/attribution.md#promptkit),
|
See the canonical [PromptKit attribution](../oss/attribution.md#promptkit),
|
||||||
[OpenRouter catalog notice](../oss/attribution.md#promptkit-openrouter-catalog),
|
[OpenRouter catalog notice](../oss/attribution.md#promptkit-openrouter-catalog),
|
||||||
and [Rakestrawhome catalog notice](../oss/attribution.md#promptkit-rakestrawhome-catalog).
|
and [Rakestrawhome catalog notice](../oss/attribution.md#promptkit-rakestrawhome-catalog).
|
||||||
|
|||||||
@@ -35,23 +35,36 @@ options before dispatch so that a missing production dependency fails as a
|
|||||||
command error rather than reaching execution.
|
command error rather than reaching execution.
|
||||||
|
|
||||||
Commands that need configuration use one shared ConfigKit-backed loader. The
|
Commands that need configuration use one shared ConfigKit-backed loader. The
|
||||||
CLI discovers the file, passes the requested overlay and only explicit
|
CLI constructs an exclusive file source for an explicit flag or environment
|
||||||
configuration-valued run overrides, and receives a **LoadedConfig**. The
|
selection, or a native required search source, then passes it to `LoadSource`
|
||||||
|
with the requested overlay and only explicit configuration-valued run overrides.
|
||||||
|
It receives a **LoadedConfig** and uses its retained main-source path for
|
||||||
|
invocation provenance without a second search. The
|
||||||
configured discovery and precedence contract is in
|
configured discovery and precedence contract is in
|
||||||
[Configuration](../config.md), while the loading and resolution mechanics are
|
[Configuration](../config.md), while the loading and resolution mechanics are
|
||||||
in [Configuration Internals](configuration.md).
|
in [Configuration Internals](configuration.md).
|
||||||
|
|
||||||
`config effective` and `config explain` only render the loaded ConfigKit view;
|
`config effective` and `config explain` render the loaded ConfigKit view by
|
||||||
they do not resolve pipelines or construct providers. Configuration validation without a selected pipeline checks structural
|
default and do not resolve pipelines or construct providers. With explicit
|
||||||
|
`--scope promptkit`, they construct the PromptKit inspection engine from the
|
||||||
|
same captured source groups used by execution. Effective output is PromptKit's
|
||||||
|
redacted YAML; explanation output shows its source history. `config profile`
|
||||||
|
uses that engine to emit one neutral Notarius-owned JSON summary for the exact
|
||||||
|
requested profile ID. These inspection commands load no run pipeline, prepare
|
||||||
|
no request, resolve no credential value, and make no provider call. Ordinary
|
||||||
|
prompt and provider-parameter content is not generally secret-free even though
|
||||||
|
credential-shaped provider parameter keys are redacted.
|
||||||
|
|
||||||
|
Configuration validation without a selected pipeline checks structural
|
||||||
configuration only. Validation with a selected pipeline also builds the
|
configuration only. Validation with a selected pipeline also builds the
|
||||||
effective catalog, resolves the pipeline, and verifies every explicit effective
|
effective catalog, resolves the pipeline, and verifies every explicit effective
|
||||||
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
|
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
|
||||||
profiles are inspected
|
profiles are inspected against the configured PromptKit source and backend
|
||||||
against the configured PromptKit source and backend registrations without
|
registrations without preparing a request or performing generation. Engine
|
||||||
loading a prompt or performing generation, so an unknown or invalid profile
|
construction validates effective prompt definitions and captures message content;
|
||||||
fails before pipeline preparation. Credential availability remains an
|
an unknown or invalid profile fails before pipeline preparation.
|
||||||
execution-time concern. Pipeline listing validates configuration before
|
Credential availability remains an execution-time concern. Pipeline listing
|
||||||
returning normalized, sorted identifiers.
|
validates configuration before returning normalized, sorted identifiers.
|
||||||
|
|
||||||
## Production Composition
|
## Production Composition
|
||||||
|
|
||||||
@@ -62,14 +75,17 @@ catalog used for resolution and the concrete constructors used for preparation.
|
|||||||
Tests may provide a catalog or registries instead; production code must not
|
Tests may provide a catalog or registries instead; production code must not
|
||||||
silently merge an injected partial catalog with production registrations.
|
silently merge an injected partial catalog with production registrations.
|
||||||
|
|
||||||
The production LLM factory builds one PromptKit-backed client from the resolved
|
The production LLM factory builds one PromptKit-backed client from the native
|
||||||
**promptkit.profile_dir** or **promptkit.profile_file** source, attaches the
|
source groups assembled from PromptKit catalogs, registered application prompt
|
||||||
profile-provenance recorder, creates one scheduler from the effective global
|
and fallback assets, the optional operator file or recursively selected
|
||||||
LLM limit, and wraps the client before it reaches modules. Registration and LLM
|
directory, and the optional local backend registration. The same construction
|
||||||
construction errors are returned before a pipeline is prepared. Configuration
|
path serves runtime, preflight, and configuration inspection. It attaches the
|
||||||
field definitions remain in [Configuration](../config.md#promptkit-profiles);
|
profile-provenance recorder, captures one immutable PromptKit configuration
|
||||||
the D&D registrar's fallback profile assets and the adapter mechanics remain in
|
identity, creates one scheduler from the effective global LLM limit, and wraps
|
||||||
[LLM Runtime](llm.md).
|
the client before it reaches modules. Registration and LLM construction errors
|
||||||
|
are returned before a pipeline is prepared. Configuration fields remain in
|
||||||
|
[Configuration](../config.md#promptkit-profiles); source ownership, capture,
|
||||||
|
identity, and adapter mechanics remain in [LLM Runtime](llm.md).
|
||||||
|
|
||||||
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
|
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
|
||||||
preserves inherit, replace, and clear states across the composition boundary.
|
preserves inherit, replace, and clear states across the composition boundary.
|
||||||
|
|||||||
@@ -15,12 +15,12 @@ runs anything.
|
|||||||
|
|
||||||
| Boundary | Inputs | Outputs | Does not own |
|
| Boundary | Inputs | Outputs | Does not own |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Loading | Selected file path, profile, overrides, and invocation directory | **LoadedConfig** with ConfigKit inspection metadata and a populated **Config** | Choosing the file path or reporting a command result. |
|
| Loading | Selected ConfigKit source, overlay, overrides, and invocation directory | **LoadedConfig** with ConfigKit inspection metadata and a populated **Config** | Choosing the source or reporting a command result. |
|
||||||
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
|
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
|
||||||
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
|
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
|
||||||
| Inspection | **LoadedConfig** and an optional JSON Pointer | Redacted effective YAML or explanation text | Pipeline selection, module preparation, or provider work. |
|
| Inspection | **LoadedConfig** and an optional JSON Pointer | Redacted effective YAML or explanation text | Pipeline selection, module preparation, or provider work. |
|
||||||
|
|
||||||
The CLI discovers a configuration file, invokes this package, and supplies the
|
The CLI describes the selected source, invokes this package, and supplies the
|
||||||
result to the framework. Configuration never reads an input file, constructs a
|
result to the framework. Configuration never reads an input file, constructs a
|
||||||
module, or creates output, cache, or debug paths. Those responsibilities remain
|
module, or creates output, cache, or debug paths. Those responsibilities remain
|
||||||
at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
||||||
@@ -28,15 +28,32 @@ at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
|||||||
|
|
||||||
## Loading And Validation
|
## Loading And Validation
|
||||||
|
|
||||||
ConfigKit acquires the selected root, imports, optional selected overlay, and
|
`Load` selects an exclusive file; `LoadSource` accepts the CLI's source and
|
||||||
explicit CLI overrides in precedence order. Its document definition supplies
|
enables imports through the same definition and callback path. ConfigKit composes
|
||||||
typed defaults, source-aware path normalization, and structural validation
|
the main input, imports, and optional overlay, then expands native inheritance
|
||||||
before conversion to the independently owned runtime **Config**. Only
|
before typed decoding. Explicit CLI assignments follow decoding and precede
|
||||||
|
application resolution. The document definition supplies typed
|
||||||
|
defaults and structural validation; Notarius's recorded resolution changes
|
||||||
|
normalize configured paths after decode. Reference paths resolve before key
|
||||||
|
normalization moves their source coordinates. Ordinary list leaf edits are
|
||||||
|
recorded at the leaf; a changed step identity requires a covering list record.
|
||||||
|
Actual map-key moves retain structural records. Path normalization follows the source
|
||||||
|
of the surviving value, including a value copied through inheritance, rather
|
||||||
|
than the inheritance directive's location. Inherited input remains explicit
|
||||||
|
for presence-sensitive behavior, including zero, false, and empty values;
|
||||||
|
existing field validation still governs whether the resulting values are valid.
|
||||||
|
ConfigKit's inspection history retains inheritance transfers alongside later
|
||||||
|
normalization, and effective configuration renders the normalized values used
|
||||||
|
by runtime conversion. The private typed document consumes the directive, so
|
||||||
|
no upstream inheritance type or unresolved directive reaches runtime pipeline
|
||||||
|
or module contracts. See [Configuration](../config.md#sibling-inheritance)
|
||||||
|
for the operator syntax and field rules. Only
|
||||||
`NOTARIUS_CONFIG` participates in discovery; retired field-level environment
|
`NOTARIUS_CONFIG` participates in discovery; retired field-level environment
|
||||||
variables do not participate in loading. Version 5 decoding rejects scalar
|
variables do not participate in loading. Version 5 decoding rejects scalar
|
||||||
module bindings and scalar external references. ConfigKit retains redacted,
|
module bindings and scalar external references. ConfigKit retains redacted,
|
||||||
historical inspection metadata while **LoadedConfig** keeps that implementation
|
historical inspection metadata while **LoadedConfig** keeps that implementation
|
||||||
detail private behind effective and explanation writers.
|
detail private behind effective and explanation writers. `MainSourcePath` reads
|
||||||
|
the non-imported main file from retained sources, and runtime conversion stays detached.
|
||||||
|
|
||||||
**Config.Validate** checks configuration-only invariants before resolution. It
|
**Config.Validate** checks configuration-only invariants before resolution. It
|
||||||
rejects incompatible profile sources, invalid state-surface values, unsupported
|
rejects incompatible profile sources, invalid state-surface values, unsupported
|
||||||
|
|||||||
@@ -72,8 +72,9 @@ An empty request profile lets the prompt select its configured default. Before a
|
|||||||
run begins, the CLI asks the adapter to inspect every explicit profile on the
|
run begins, the CLI asks the adapter to inspect every explicit profile on the
|
||||||
resolved selected LLM-backed bindings and validators, including inherited
|
resolved selected LLM-backed bindings and validators, including inherited
|
||||||
pipeline profiles. Inspection resolves the profile and its selected backend and
|
pipeline profiles. Inspection resolves the profile and its selected backend and
|
||||||
target without loading a prompt, reading credentials, admitting capacity, or
|
target without preparing a request, reading credentials, admitting capacity, or
|
||||||
contacting a provider, so a missing or invalid explicit profile fails before
|
contacting a provider. Engine construction validates effective prompt definitions
|
||||||
|
and captures message files and static templates. A missing or invalid profile fails before
|
||||||
stage execution while a valid `api_key_env` may remain unset. Calls record the
|
stage execution while a valid `api_key_env` may remain unset. Calls record the
|
||||||
profile actually selected by PromptKit. The recorder trims and deduplicates
|
profile actually selected by PromptKit. The recorder trims and deduplicates
|
||||||
non-secret profile identity, provider, model, selected backend ID, and
|
non-secret profile identity, provider, model, selected backend ID, and
|
||||||
@@ -83,35 +84,43 @@ retain an empty backend ID, which the published JSON omits. Successful
|
|||||||
completion responses and recorded profile manifests identify the adapter
|
completion responses and recorded profile manifests identify the adapter
|
||||||
provider as `promptkit`.
|
provider as `promptkit`.
|
||||||
|
|
||||||
The CLI's profile-inspection engine and the production adapter use the same
|
The CLI's profile-inspection engine and production adapter use the same source
|
||||||
profile-source construction to apply the configured profile directory or file,
|
assembly: PromptKit catalogs, one application group containing registered
|
||||||
the optional registered fallback profile assets, and the optional conventional
|
prompts and fallback profiles, the selected operator file or directory, and
|
||||||
`local` backend. Preflight therefore resolves the same profile sources and
|
the optional final `local` backend registration. Execution, preflight, and
|
||||||
backend membership as runtime without performing generation. Fallback assets
|
inspection therefore capture the same effective definitions. PromptKit and
|
||||||
are mounted only when at least one source is registered. The production D&D
|
ConfigKit compose recursively across groups while retaining each input's file
|
||||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
context; inputs within a group compose strictly. Operator imports are enabled
|
||||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
for the selected operator input, and imports retain their declaring file's
|
||||||
profile parsing and inheritance: an operator-provided matching profile takes precedence over a
|
relative resource roots. PromptKit validates final effective definitions at
|
||||||
fallback profile without Notarius merging either document.
|
engine construction. It does not run a separate audit that rejects a malformed
|
||||||
When the registration is absent, a profile selecting `backend: local` fails
|
definition superseded by a later overlay. The D&D registrar contributes its
|
||||||
inspection instead of falling back to a built-in or endpoint-only target.
|
`dnd-extraction` fallback, and maintained D&D prompts select that logical ID
|
||||||
|
by default. A profile selecting `backend: local` can use an operator
|
||||||
|
`backends.local` declaration or the optional Notarius registration; see
|
||||||
|
[native source rules](../config.md#native-promptkit-sources-and-migration).
|
||||||
|
|
||||||
Before execution, the adapter also contributes a non-secret checkpoint
|
This PromptKit integration uses the old-path ConfigKit v0.5.0 types exposed by
|
||||||
fingerprint for the effective PromptKit profile source. It combines the
|
PromptKit v0.12.0. Application configuration uses the distinct new-path
|
||||||
identity of PromptKit's compiled-in profile catalog with a deterministic digest
|
ConfigKit package; those named types do not cross the adapter boundary.
|
||||||
of every YAML profile in the configured profile directory, or of the configured
|
|
||||||
profile file, and a deterministic digest of the flattened fallback profile
|
Each engine captures its definitions at construction. A later source-file edit
|
||||||
assets. The fingerprint contains neither profile content nor source paths. It
|
does not mutate an existing engine; constructing a new engine captures the
|
||||||
covers inherited pipeline profiles, explicit binding profiles, and
|
updated view. Before execution, the adapter computes one immutable,
|
||||||
prompt-selected defaults, so changing a model or other profile setting cannot
|
versioned `promptkit_configuration` checkpoint identity from public captured
|
||||||
reuse checkpoints created under the
|
inspection APIs. The private digest covers captured effective configuration,
|
||||||
prior profile source. This cache identity is independent of durable
|
all effective profile targets and credential-policy metadata (including
|
||||||
profile provenance: run manifests continue to list only profiles actually
|
provider parameters), and every effective prompt ID/version and captured
|
||||||
observed during LLM calls. When the local backend is registered, a second
|
content hash. The identity also includes PromptKit and the selected OpenRouter
|
||||||
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
|
and Rakestrawhome catalog versions. Local-backend concurrency settings remain
|
||||||
semantic execution target invalidates checkpoint reuse. The raw endpoint is not
|
excluded because they affect scheduling; its endpoint is represented only in
|
||||||
stored in checkpoint identity, and the local concurrency limit is excluded
|
the private digest, never as checkpoint metadata. Neither the private digest
|
||||||
because it changes scheduling rather than execution semantics.
|
payload, raw sources, paths, endpoints, provider parameters, nor prompt bodies
|
||||||
|
are published. An existing engine retains its identity after files change; a
|
||||||
|
new engine gets the identity of its newly captured definitions. This replaces
|
||||||
|
the former live-file profile and local-endpoint fingerprints. Run manifests
|
||||||
|
remain independent and list only profile choices actually observed during LLM
|
||||||
|
calls.
|
||||||
|
|
||||||
## Shared Provider-Call Limit
|
## Shared Provider-Call Limit
|
||||||
|
|
||||||
@@ -140,11 +149,16 @@ only by the Notarius scheduler.
|
|||||||
## Prompt And Schema Assets
|
## Prompt And Schema Assets
|
||||||
|
|
||||||
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
|
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
|
||||||
filesystems from production module families. It flattens registered roots into
|
filesystems from production module families. It retains each registered root's
|
||||||
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
|
filesystem context when constructing PromptKit's application source group.
|
||||||
assets, duplicate paths, and missing prompt or schema files during preparation.
|
Schema files are supplied separately through the engine's schema filesystem;
|
||||||
Fallback assets receive a safe content digest for checkpoint identity; raw
|
PromptKit reads and validates a selected schema while preparing that request,
|
||||||
paths and bytes are never included. The framework’s `promptfs` helper combines
|
not while constructing the configuration snapshot. Prompt and fallback
|
||||||
|
definitions are validated after native source composition, and referenced
|
||||||
|
prompt content is captured with the engine. Checkpoint identity uses captured
|
||||||
|
prompt content hashes rather than rescanning live files or hashing only
|
||||||
|
content-file paths. Raw paths and bytes are never included. The framework’s
|
||||||
|
`promptfs` helper combines
|
||||||
module-selected prompt files with reusable domain fragments without making the
|
module-selected prompt files with reusable domain fragments without making the
|
||||||
framework depend on D&D content.
|
framework depend on D&D content.
|
||||||
|
|
||||||
@@ -294,6 +308,10 @@ When debug recording is enabled, the pipeline decorates the shared client. The
|
|||||||
wrapper records prepared prompt and response material, timing, selected profile
|
wrapper records prepared prompt and response material, timing, selected profile
|
||||||
and backend, effective model parameters, and call identifiers in the run’s
|
and backend, effective model parameters, and call identifiers in the run’s
|
||||||
debug bundle, including material available from a failed structured completion.
|
debug bundle, including material available from a failed structured completion.
|
||||||
|
The profile-inspection CLI projects copied values into its Notarius-owned
|
||||||
|
neutral JSON DTO; effective configuration and explanation remain PromptKit's
|
||||||
|
redacted YAML and source-history views. Credential names and `APIKeyRequired`
|
||||||
|
metadata are configuration metadata, not credential-availability checks.
|
||||||
Effective parameters use PromptKit's stable lower-case JSON field names and may
|
Effective parameters use PromptKit's stable lower-case JSON field names and may
|
||||||
include `backend_id`. For a successful completion, a debug-write failure is
|
include `backend_id`. For a successful completion, a debug-write failure is
|
||||||
surfaced; when the completion already failed, its call error remains the
|
surfaced; when the completion already failed, its call error remains the
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ who need implementation mechanics should read [Run State Internals](internal/sta
|
|||||||
## Source Deployment
|
## Source Deployment
|
||||||
|
|
||||||
Linux is the supported deployment platform. Install a pinned source release
|
Linux is the supported deployment platform. Install a pinned source release
|
||||||
with the Go version declared in `go.mod` (currently Go 1.25.5):
|
with the Go version declared in `go.mod` (currently Go 1.27.0):
|
||||||
|
|
||||||
~~~sh
|
~~~sh
|
||||||
GOWORK=off go install \
|
GOWORK=off go install \
|
||||||
@@ -99,24 +99,25 @@ Profile deployment has four distinct layers:
|
|||||||
|
|
||||||
| Layer | Owner | Operational role |
|
| Layer | Owner | Operational role |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Prompts and schemas | Notarius module families | Embedded request and structured-output definitions. They are not deployment profile files. |
|
| Prompts and schemas | Notarius module families, with optional operator prompt overlays | Embedded request and structured-output definitions; operator configuration may recursively overlay prompt fields. Schemas remain application assets. |
|
||||||
| Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. |
|
| Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. |
|
||||||
| Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. |
|
| Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. |
|
||||||
| Operator profiles | Deployment filesystem | Complete environment-specific definitions selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
|
| Operator configuration | Deployment filesystem | Native profile/backend definitions and prompt overlays selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
|
||||||
|
|
||||||
The maintained D&D pipeline uses the workload ID `dnd-extraction`. The
|
The maintained D&D pipeline uses the workload ID `dnd-extraction`. The
|
||||||
embedded fallback makes that ID usable without an operator file. Production,
|
embedded fallback makes that ID usable without an operator file. Production,
|
||||||
development, and local deployments can each install a different complete
|
development, and local deployments can each overlay settings for the same ID,
|
||||||
definition for the same ID, retaining the pipeline while choosing their own
|
retaining inherited values they do not change. Notarius's registered fallback
|
||||||
model, backend, timeout, or reasoning policy. An operator definition wins over
|
configuration precedes the selected operator source, and groups compose by
|
||||||
the fallback; it is not merged with it. The configuration field and full
|
native recursive overlays; a configured Notarius local backend is applied
|
||||||
precedence rules are owned by [Configuration](config.md#promptkit-profiles).
|
after the operator source. The configuration fields and source precedence are
|
||||||
|
owned by [Configuration](config.md#promptkit-profiles).
|
||||||
|
|
||||||
Use a profile source owned by the service account, keep it readable only by
|
Use a profile source owned by the service account, keep it readable only by
|
||||||
the intended operator, and supply provider credentials through the service
|
the intended operator, and supply provider credentials through the service
|
||||||
environment—not in the Notarius configuration or profile YAML. The maintained
|
environment—not in the Notarius configuration or profile YAML. The maintained
|
||||||
[LLM profile](../examples/llm-profiles/dnd-extraction.yml) is secret-free and
|
[LLM profiles](../examples/llm-profiles/dnd-extraction.yml) are secret-free
|
||||||
can be copied as a format starting point. Validate a deployment without a
|
and can be copied as a format starting point. Validate a deployment without a
|
||||||
provider call or credentials:
|
provider call or credentials:
|
||||||
|
|
||||||
~~~sh
|
~~~sh
|
||||||
@@ -132,6 +133,40 @@ surviving declaration. The complete example's
|
|||||||
Use absolute paths such as `/etc/notarius/profiles/dnd-extraction.yml` for
|
Use absolute paths such as `/etc/notarius/profiles/dnd-extraction.yml` for
|
||||||
services and containers when that is clearer for the deployment.
|
services and containers when that is clearer for the deployment.
|
||||||
|
|
||||||
|
### Inspect Before Deployment
|
||||||
|
|
||||||
|
Use the three configuration reports together when reviewing an operator
|
||||||
|
source:
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
notarius config effective --scope promptkit --config /etc/notarius/config.yml
|
||||||
|
notarius config explain --scope promptkit --config /etc/notarius/config.yml \
|
||||||
|
/profiles/dnd-extraction/model
|
||||||
|
notarius config profile --config /etc/notarius/config.yml dnd-extraction
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Each report command constructs an inspection engine from the selected
|
||||||
|
configuration and registered application assets. Construction validates and
|
||||||
|
captures the effective definitions; after editing a source, run the command
|
||||||
|
again to inspect a newly constructed engine. These commands do not resolve
|
||||||
|
credential values, call a provider, or prepare a request. `config profile`
|
||||||
|
reports the named profile's resolved target and effective credential metadata,
|
||||||
|
not whether a credential is available or which profile a particular prompt or
|
||||||
|
run will select.
|
||||||
|
|
||||||
|
Effective and explanation reports can include ordinary inline prompt and
|
||||||
|
provider-parameter data. Credential-shaped provider parameter keys are
|
||||||
|
redacted, but the rest of the content is not treated as secret-free. Reports
|
||||||
|
are written only when requested; Notarius does not automatically persist them
|
||||||
|
in run output or debug bundles. Operators own their destination, access, and
|
||||||
|
retention when redirecting or storing reports.
|
||||||
|
|
||||||
|
Configuration reports are distinct from a prepared execution or a requested
|
||||||
|
debug trace. They describe captured definitions and source history, not
|
||||||
|
rendered request messages, runtime credential values, or an individual
|
||||||
|
provider interaction. The [CLI reference](cli.md#config-effective) documents
|
||||||
|
the report syntax and output contracts.
|
||||||
|
|
||||||
## Run Lifecycle
|
## Run Lifecycle
|
||||||
|
|
||||||
Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads
|
Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads
|
||||||
|
|||||||
@@ -11,14 +11,14 @@ Copyright (C) 2026 Eric Rakestraw
|
|||||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||||
|
|
||||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4).
|
||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.4.2`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -26,14 +26,29 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||||
[GPL 3.0](licenses/GPL-3.0.txt) |
|
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e).
|
||||||
|
|
||||||
|
## ConfigKit (old module path)
|
||||||
|
|
||||||
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
||||||
|
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
||||||
|
|
||||||
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||||
|
|
||||||
## PromptKit
|
## PromptKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.11.2`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.12.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -41,38 +56,38 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL 3.0](licenses/LGPL-3.0.txt) |
|
||||||
[GPL 3.0](licenses/GPL-3.0.txt) |
|
[GPL 3.0](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3).
|
||||||
|
|
||||||
## PromptKit OpenRouter Catalog
|
## PromptKit OpenRouter Catalog
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` at `v2.20260926.0`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
License and notices: [MIT](licenses/MIT.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
||||||
|
|
||||||
## PromptKit Rakestrawhome Catalog
|
## PromptKit Rakestrawhome Catalog
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` at `v2.20260926.0`.
|
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
License and notices: [MIT](licenses/MIT.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
||||||
|
|
||||||
## jsonschema
|
## jsonschema
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/agents.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/agents.md
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/architecture.md
|
||||||
|
|
||||||
## Purpose And Ownership
|
## Purpose And Ownership
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Documentation Policy
|
# Documentation Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.11
|
**Base Policy Version:** 1.12
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/documentation.md
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -86,8 +86,8 @@ secret values.
|
|||||||
| Integration contracts for external components (if applicable) | `docs/integrations/` | External formats and protocols, upstream and downstream contracts, logical output paths and schemas, media types, and compatibility. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
| Integration contracts for external components (if applicable) | `docs/integrations/` | External formats and protocols, upstream and downstream contracts, logical output paths and schemas, media types, and compatibility. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
| Architectural decision records | `docs/adr/` | Records of significant architectural decisions and supporting rationale. | Ordinary design choices, current behavior reference, implementation status, task sequencing. |
|
||||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, or future work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
|
|
||||||
@@ -153,8 +153,15 @@ Use sequentially numbered ADR filenames such as
|
|||||||
|
|
||||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
|
||||||
belong in the roadmap.
|
ADRs should be reserved for major architectural decisions, such as ownership
|
||||||
|
boundaries, large-scale architectural features, or invariants that rise to
|
||||||
|
the level of architectural significance. Routine choices, implementation
|
||||||
|
decisions, and rejected suggestions do not warrant an ADR unless they
|
||||||
|
otherwise represent a significant architectural decision.
|
||||||
|
|
||||||
|
Roadmaps should preserve plausible future work, not rejected product
|
||||||
|
ideas.
|
||||||
|
|
||||||
## Maintenance
|
## Maintenance
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.0
|
**Base Policy Version:** 1.0
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/meta.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/meta.md
|
||||||
|
|
||||||
## Origin And Local Authority
|
## Origin And Local Authority
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Open Source Licensing And Attribution Policy
|
# Open Source Licensing And Attribution Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.8
|
**Base Policy Version:** 1.9
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/oss.md
|
||||||
|
|
||||||
## Purpose And Scope
|
## Purpose And Scope
|
||||||
|
|
||||||
@@ -67,6 +67,9 @@ individual packages.
|
|||||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||||
to create the project’s root `COPYRIGHT` file.
|
to create the project’s root `COPYRIGHT` file.
|
||||||
|
Use the COPYRIGHT examples exactly as provided, except for placeholder
|
||||||
|
substitutions and changes expressly required or authorized by this policy or
|
||||||
|
the author.
|
||||||
Use verified project names, contribution years, and copyright holders.
|
Use verified project names, contribution years, and copyright holders.
|
||||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||||
text references together to reflect the author's selected grant.
|
text references together to reflect the author's selected grant.
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Release Policy
|
# Release Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.4
|
**Base Policy Version:** 1.7
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/release.md
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
|||||||
intended source revision under a fresh version. Releases should require little
|
intended source revision under a fresh version. Releases should require little
|
||||||
routine intervention, avoid repeated work, and preserve published contents.
|
routine intervention, avoid repeated work, and preserve published contents.
|
||||||
|
|
||||||
|
Adopt stronger verification or reproducibility requirements through an explicit
|
||||||
|
project-specific decision justified by the intended audience and support needs.
|
||||||
|
|
||||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||||
policy and link to it rather than introduce additional release requirements.
|
policy and link to it rather than introduce additional release requirements.
|
||||||
@@ -52,9 +55,14 @@ requires a deliberate update to the policy and procedure where needed.
|
|||||||
For releases at or after `v1.0.0`:
|
For releases at or after `v1.0.0`:
|
||||||
|
|
||||||
- a major release may make intentional incompatible changes;
|
- a major release may make intentional incompatible changes;
|
||||||
- a minor release may add backward-compatible behavior; and
|
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||||
|
minimum as described below; and
|
||||||
- a patch release may make backward-compatible fixes.
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
|
In this context, "backward-compatible" means only that pre-existing, valid
|
||||||
|
configurations will continue to work following the relase. Behavior need not
|
||||||
|
remain strictly identical.
|
||||||
|
|
||||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
configuration, data, integration, or compatibility contract when the release
|
configuration, data, integration, or compatibility contract when the release
|
||||||
note explains the impact and required consumer action. A patch release must
|
note explains the impact and required consumer action. A patch release must
|
||||||
@@ -64,6 +72,36 @@ Additional compatibility promises and deliberate exceptions belong in the
|
|||||||
adopted policy or canonical contract documentation. Release notes explain
|
adopted policy or canonical contract documentation. Release notes explain
|
||||||
changes; they do not redefine current compatibility contracts.
|
changes; they do not redefine current compatibility contracts.
|
||||||
|
|
||||||
|
## Consumer Minimum And Release Toolchain
|
||||||
|
|
||||||
|
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||||
|
maintainers use to build and validate a release. Record the consumer minimum in
|
||||||
|
the language's standard metadata and explain its support meaning in canonical
|
||||||
|
documentation. Define the selected release toolchain and its setup commands in
|
||||||
|
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||||
|
choices.
|
||||||
|
|
||||||
|
For Go repositories, normally declare the initial stable release of the selected
|
||||||
|
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||||
|
when a dependency or an identified correctness fix requires it, and document
|
||||||
|
the reason.
|
||||||
|
|
||||||
|
For release-policy purposes, successful build and release validation on the
|
||||||
|
selected Go patch release are sufficient to presume support across that Go
|
||||||
|
series, subject to the declared consumer minimum and concrete evidence of
|
||||||
|
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||||
|
|
||||||
|
Use the selected release toolchain for candidate validation and release builds,
|
||||||
|
including CI jobs performing those tasks. A newer validation patch version alone
|
||||||
|
leaves the consumer minimum unchanged.
|
||||||
|
|
||||||
|
A minor release may raise the Go consumer minimum, including from one Go series
|
||||||
|
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||||
|
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||||
|
the consumer minimum. Explain an increase's impact and required consumer action
|
||||||
|
in the release note. Other incompatible changes remain subject to the versioning
|
||||||
|
rules above.
|
||||||
|
|
||||||
## Release Documentation And Preparation
|
## Release Documentation And Preparation
|
||||||
|
|
||||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||||
@@ -191,7 +229,8 @@ release without replacing its contents or reusing its version.
|
|||||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||||
identify:
|
identify:
|
||||||
|
|
||||||
1. prerequisites, primary branch, and authoritative remote;
|
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||||
|
remote;
|
||||||
2. version and release-note preparation;
|
2. version and release-note preparation;
|
||||||
3. one publication command with its selected local checks and automated guards;
|
3. one publication command with its selected local checks and automated guards;
|
||||||
4. source-only delivery, required build targets, and automation responsibilities;
|
4. source-only delivery, required build targets, and automation responsibilities;
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-17
|
||||||
|
|
||||||
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/testing.md
|
**Upstream Source**: https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/testing.md
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,22 @@ for development, and Windows is unsupported. `main` on `origin` is authoritative
|
|||||||
|
|
||||||
## Prepare
|
## Prepare
|
||||||
|
|
||||||
Use Git, a POSIX shell, and Go 1.25.5 or newer. Configure ordinary Git credentials
|
Use Git, a POSIX shell, and a Go installation supporting toolchain selection.
|
||||||
for `origin` and private-module access without putting secrets in the repository.
|
The selected release toolchain is Go 1.27.0. Select it for candidate validation
|
||||||
Prepare dependencies before offline validation where necessary:
|
and release builds, including any CI performing those tasks:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export GOTOOLCHAIN=go1.27.0
|
||||||
|
go version
|
||||||
|
```
|
||||||
|
|
||||||
|
The consumer minimum is declared in [`go.mod`](../go.mod). Its patch-level
|
||||||
|
minimum is required by the selected ConfigKit dependency. Selecting a newer
|
||||||
|
release toolchain is a separate runbook update and leaves that minimum unchanged.
|
||||||
|
|
||||||
|
Configure ordinary Git credentials for `origin` and private-module access
|
||||||
|
without putting secrets in the repository. Prepare the selected toolchain and
|
||||||
|
dependencies before offline validation where necessary:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
GOWORK=off GOPRIVATE=gitea.maximumdirect.net go mod download
|
GOWORK=off GOPRIVATE=gitea.maximumdirect.net go mod download
|
||||||
@@ -39,9 +52,10 @@ Run one command from the repository with the actual selected version:
|
|||||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||||
```
|
```
|
||||||
|
|
||||||
The command checks version availability on `origin`, branch and worktree state,
|
Both release scripts inherit the `GOTOOLCHAIN` selected during preparation.
|
||||||
and the matching note. It runs the complete offline default test suite once and
|
The publication command checks version availability on `origin`, branch and
|
||||||
a native CLI build/version check with `GOWORK=off`. The
|
worktree state, and the matching note. It runs the complete offline default test
|
||||||
|
suite once and a native CLI build/version check with `GOWORK=off`. The
|
||||||
[gate rationale](policy/release.md#notarius-gate-selection) explains the selected
|
[gate rationale](policy/release.md#notarius-gate-selection) explains the selected
|
||||||
checks. It then rechecks candidate and publication state, pushes the exact
|
checks. It then rechecks candidate and publication state, pushes the exact
|
||||||
candidate to `main` without force, creates a lightweight tag, and pushes only
|
candidate to `main` without force, creates a lightweight tag, and pushes only
|
||||||
|
|||||||
244
docs/roadmap/promptkit.md
Normal file
244
docs/roadmap/promptkit.md
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
# PromptKit v0.23.2 Adoption
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Replace Notarius's PromptKit v0.12 integration with a native v0.23.2
|
||||||
|
integration. Delegate configuration composition, resource inspection, request
|
||||||
|
preparation, and model execution to PromptKit and ConfigKit. Keep application
|
||||||
|
machinery where it implements a distinct Notarius decision, such as typed
|
||||||
|
artifact handling, stage acceptance, cross-backend scheduling, or checkpoint
|
||||||
|
reuse.
|
||||||
|
|
||||||
|
This is a hard cutover for both the Notarius application configuration and
|
||||||
|
PromptKit resource definitions. Provide migration guidance and maintained
|
||||||
|
examples; support only the new formats in the resulting application. This
|
||||||
|
draft describes the target, assuming the recommendations under
|
||||||
|
[Open Decisions](#open-decisions). It does not select a release date or
|
||||||
|
implementation sequence.
|
||||||
|
|
||||||
|
## Reviewed Baseline
|
||||||
|
|
||||||
|
Notarius at `65b8e8f0a2d60041c18a29a3c33afc5c689468df` uses application
|
||||||
|
configuration version 5, Go 1.27, new-path ConfigKit v1.2.0, and old-path
|
||||||
|
PromptKit v0.12.0. The latter brings in a separate old-path ConfigKit v0.5.0
|
||||||
|
and two external provider catalogs. Notarius and PromptKit currently share
|
||||||
|
the YAML v4 release candidate.
|
||||||
|
|
||||||
|
The target is `go.maximumdirect.net/promptkit@v0.23.2`, published at
|
||||||
|
`382ccfb14b9630fd8f48c8364dbe2b6159637d71`. The tagged [release
|
||||||
|
notes][release], [migration guide][migration], [format reference][formats],
|
||||||
|
and [Go consumer guide][consumer] are the primary upstream references.
|
||||||
|
|
||||||
|
The main baseline seams are:
|
||||||
|
|
||||||
|
- `internal/core/config` resolves `promptkit.profile_file` and
|
||||||
|
`profile_dir` and interprets the optional `local_backend` object.
|
||||||
|
`promptkit_profiles.go` builds the operator layers for execution and
|
||||||
|
inspection; `asset_registry.go` supplies the embedded layer.
|
||||||
|
- `assets/` and `examples/llm-profiles/` use keyed resource definitions.
|
||||||
|
Completion requests and diagnostics carry a PromptKit prompt version.
|
||||||
|
- `internal/framework/llm` does split preparation and execution to retain
|
||||||
|
debug material; it projects a backend-completed profile report and walks
|
||||||
|
effective YAML to discover versioned prompts for checkpoint identity.
|
||||||
|
- The pipeline's producer-attempt loop coordinates domain and stage policy.
|
||||||
|
The scheduled LLM client enforces the application-wide limit independently
|
||||||
|
of any PromptKit backend policy.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
### Dependency And Application Configuration Cutover
|
||||||
|
|
||||||
|
Use the vanity PromptKit import path and shared ConfigKit v1.2.1, stable
|
||||||
|
go-yaml v4, and the resulting compatible validation dependencies. Remove the
|
||||||
|
old-path imports and catalog dependencies once unused. Use tagged modules
|
||||||
|
without a compatibility fork or permanent replacement. The consumer minimum
|
||||||
|
remains Go 1.27. Update attribution for the selected and removed components
|
||||||
|
under the [OSS policy](../policy/oss.md).
|
||||||
|
|
||||||
|
Introduce application configuration version 6 as the only supported application
|
||||||
|
version. Version 6 retains unrelated pipeline structure, discovery, and
|
||||||
|
resolution behavior. ConfigKit continues to own application loading and
|
||||||
|
composition under [ADR-0020](../adr/0020-delegate-configuration-mechanics-to-configkit.md).
|
||||||
|
Retire `profile_file`, `profile_dir`, and `local_backend` in favor of native
|
||||||
|
PromptKit companion files. There is no separate `promptkit.config_file` selector.
|
||||||
|
Preserve the ability to override transport retries through the PromptKit engine
|
||||||
|
option; leave the upstream default in effect when the operator does not
|
||||||
|
override it.
|
||||||
|
|
||||||
|
### Companion Files And Embedded Defaults
|
||||||
|
|
||||||
|
Adopt Audita's companion-file convention. Use the directory of the main
|
||||||
|
Notarius configuration actually selected by ConfigKit, whether explicit or
|
||||||
|
discovered. Load the optional `prompts.yml` and recursive `.yml` and `.yaml`
|
||||||
|
files under `prompts/` beside it as one higher-priority operator underlay
|
||||||
|
using native ConfigKit imports. Do not add a working-directory fallback
|
||||||
|
for companion discovery. Configurations in the same directory share those
|
||||||
|
companions. Missing companions are fine; loading errors follow upstream
|
||||||
|
semantics rather than silently falling back to embedded assets. Native
|
||||||
|
imports and source-relative resource paths govern any files reached from the
|
||||||
|
companions. The two companion locations are strict peers; filenames do not
|
||||||
|
set precedence between conflicting operator records.
|
||||||
|
|
||||||
|
Use one source assembly for runtime, preflight, and inspection: PromptKit's
|
||||||
|
bundled OpenRouter underlay, Notarius's embedded underlay, then the operator
|
||||||
|
underlay when present. Embed all Notarius default prompts and profiles under
|
||||||
|
`collections.notarius` in the application underlay. Operator companion files
|
||||||
|
can replace individual defaults by defining complete records with the same
|
||||||
|
local IDs in that same collection. They can also define new resources and
|
||||||
|
local backends using the native format.
|
||||||
|
|
||||||
|
The bundled OpenRouter inventory differs from the former external catalogs
|
||||||
|
and supplies no backend-local concurrency limit. It does not implicitly
|
||||||
|
select a model for any Notarius request. Migrate deployments that use removed
|
||||||
|
profiles or backends to intentionally selected native records. The
|
||||||
|
application-wide LLM ceiling remains independent of backend-local settings.
|
||||||
|
|
||||||
|
### Native Prompt And Profile Resources
|
||||||
|
|
||||||
|
Convert maintained prompts, profiles, examples, and fixtures to identified
|
||||||
|
lists, with `config:` for source controls and native sibling inheritance
|
||||||
|
(`extends: {id: parent}`). Under native default composition, a later same-ID
|
||||||
|
record replaces the earlier record in full. Explain complete replacements,
|
||||||
|
distinct inherited variants, and the new backend fallback semantics for
|
||||||
|
profile nulls.
|
||||||
|
The optional PromptKit source specification header is version 2; it is not the
|
||||||
|
Notarius application version. Do not translate legacy resource files,
|
||||||
|
synthesize aliases, or reconstruct old recursive same-ID overlays.
|
||||||
|
|
||||||
|
Select prompts by ID alone and remove the obsolete PromptKit version selector
|
||||||
|
and metadata from Notarius's request, debug, and reconciliation identity
|
||||||
|
plumbing. Review affected diagnostic fields and document deliberate changes.
|
||||||
|
Keep the current local resource IDs and use their qualified forms in
|
||||||
|
lookups and references, for example `notarius/dnd.spells` and
|
||||||
|
`notarius/dnd-extraction`. Update prompt profile references, application
|
||||||
|
profile selections, and inspection examples to use the qualified IDs.
|
||||||
|
The `dnd-extraction` profile retains its current model and settings and
|
||||||
|
continues to reference the unqualified bundled `openrouter` backend. A
|
||||||
|
profile that extends it belongs in the same collection and uses its local
|
||||||
|
sibling ID in `extends`. Retain message roles, bytes, order, cache boundaries,
|
||||||
|
and intended output/repair behavior while changing the declaration format.
|
||||||
|
Keep module-owned asset selection and embedded filesystem contexts intact.
|
||||||
|
Verify maintained private response schemas work under upstream's
|
||||||
|
self-contained-schema rule; this is not a redesign of durable D&D artifacts.
|
||||||
|
|
||||||
|
### Inspection And Reuse Identity
|
||||||
|
|
||||||
|
Use PromptKit's captured configuration reports for effective configuration
|
||||||
|
and provenance. Use its resource inventory and inspection APIs for exact
|
||||||
|
identities, presence-aware resource fields, and captured prompt hashes, rather
|
||||||
|
than deriving an inventory from rendered configuration YAML. Change the
|
||||||
|
`config profile` report to show the resolved profile and its selected backend,
|
||||||
|
when present, as separate resources. Use upstream field semantics and retain
|
||||||
|
the existing operator disclosure boundary, including redaction of
|
||||||
|
credential-shaped provider parameters. Do not recreate backend default
|
||||||
|
resolution to produce the former `effective_model_params` view.
|
||||||
|
|
||||||
|
Distinguish configuration and resource preflight from request readiness.
|
||||||
|
Use `InspectRequirements` to explain recognized dependencies when helpful,
|
||||||
|
not as proof that a request is executable. Use `Prepare` or
|
||||||
|
`ValidateRequest` when concrete request inputs are available and their
|
||||||
|
validation is actually needed. A routine config-only check should remain
|
||||||
|
credential-free and offline, without synthetic requests.
|
||||||
|
|
||||||
|
Retain a private, versioned Notarius checkpoint identity based on the engine's
|
||||||
|
captured state. Use native resource enumeration and prompt hashes, keeping
|
||||||
|
the effective execution inputs needed to distinguish reuse safely. A diff of
|
||||||
|
effective values alone cannot detect edits to a referenced message file.
|
||||||
|
Version the fingerprint for the cutover so obsolete identity inputs cannot
|
||||||
|
cause a false hit; do not reread mutable sources or expose the digest inputs.
|
||||||
|
Keep run manifests based on the profiles actually used, not the inventory.
|
||||||
|
|
||||||
|
### Native Execution And Retained Pipeline Policy
|
||||||
|
|
||||||
|
Use ordinary `Engine.Run` for structured completion and use `OnPrepared`
|
||||||
|
where opt-in debug capture needs the actual prepared request. The callback
|
||||||
|
runs after admission, before generation, and is not delivered for failed
|
||||||
|
preparation or admission. Keep the callback lightweight and use native
|
||||||
|
observations for safe terminal execution facts, including attempts, repairs,
|
||||||
|
and available token usage. Where a diagnostic needs totals over selected
|
||||||
|
executions, use `ExecutionGroup` rather than inventing a second accumulator.
|
||||||
|
Use `OperationTimeout` where a completion calls for that scope of budget;
|
||||||
|
do not add a new default deadline or replace application deadlines that also
|
||||||
|
cover work outside the PromptKit call.
|
||||||
|
|
||||||
|
PromptKit owns provider admission, transport retries, and structural repair.
|
||||||
|
Keep the Notarius cross-backend scheduler because it serves a different
|
||||||
|
budget from backend or pipeline worker limits. Keep the adapter's typed
|
||||||
|
decode, actual-run provenance, and provider-neutral error and diagnostic
|
||||||
|
mapping. PromptKit execution types stay within that integration boundary.
|
||||||
|
In particular, preserve the distinction between invalid structured
|
||||||
|
output, provider failure, capacity rejection, and cancellation. A configured
|
||||||
|
repair override still selects a complete upstream output contract; it must not
|
||||||
|
accidentally drop the selected schema or validation mode.
|
||||||
|
|
||||||
|
Notarius owns stage attempts, validation and fallback decisions, and
|
||||||
|
publication of accepted artifacts. Keep its latest-response-only correction
|
||||||
|
policy using native appended messages, as defined by
|
||||||
|
[ADR-0014](../adr/0014-feedback-aware-validation-retries.md) and
|
||||||
|
[ADR-0016](../adr/0016-feedback-aware-module-requested-retries.md).
|
||||||
|
`Exchange` and `Coordinate` are for workflows that benefit from retained
|
||||||
|
conversation history. Adopting them for stage retries would change what the
|
||||||
|
model sees and which work consumes an attempt. They are not needed to execute
|
||||||
|
the existing retry policy.
|
||||||
|
|
||||||
|
## Migration And Verification
|
||||||
|
|
||||||
|
Update the maintained minimal and complete D&D examples to distinguish
|
||||||
|
embedded-only operation from companion configuration. Use separate example
|
||||||
|
directories if necessary to demonstrate those deployments independently.
|
||||||
|
Document how to move existing PromptKit sources into the companion layout
|
||||||
|
or import them from it, declare a local backend, and replace or extend an
|
||||||
|
embedded resource. Explain qualified IDs, retired catalog IDs, the changed
|
||||||
|
profile report, and any new checkpoint cold misses. Update the canonical
|
||||||
|
configuration, CLI, operations, and integration guides when behavior lands.
|
||||||
|
Record the changed decisions about source precedence and inspection in a
|
||||||
|
successor to
|
||||||
|
[ADR-0019](../adr/0019-adopt-native-promptkit-configuration.md) rather than
|
||||||
|
rewriting its accepted decision text. Release preparation is separate.
|
||||||
|
|
||||||
|
Verify the dependency graph and offline examples after the cutover. Use
|
||||||
|
focused behavior tests for the new format and intentional rejection of the
|
||||||
|
old one, main-relative companion discovery, and native relative imports.
|
||||||
|
Verify that a companion can replace a qualified embedded prompt or profile
|
||||||
|
without changing unrelated defaults, and that preflight, inspection, and
|
||||||
|
runtime use the same effective definitions. Exercise preparation,
|
||||||
|
generation, structural repair, and semantic retries through the maintained
|
||||||
|
fake-provider integration. Check that real message or target changes affect
|
||||||
|
checkpoint reuse, and that session routing, cache controls, diagnostics, and
|
||||||
|
durable outputs keep their existing meaning. Live provider checks are useful
|
||||||
|
for deployment confidence, not a requirement of the offline suite.
|
||||||
|
|
||||||
|
[release]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/releases/v0.23.2.md
|
||||||
|
[migration]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/consumers/configuration-migration.md
|
||||||
|
[formats]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/formats.md
|
||||||
|
[consumer]: https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/consumers/pkg-promptkit.md
|
||||||
|
|
||||||
|
## Open Decisions
|
||||||
|
|
||||||
|
### How Far Offline Preflight Goes
|
||||||
|
|
||||||
|
**Recommendation:** Check that the captured configuration is valid and that
|
||||||
|
selected resource IDs resolve. Make clear that a valid profile can be partial
|
||||||
|
and that concrete request readiness is checked during preparation. This
|
||||||
|
preserves a useful offline preflight without duplicating target resolution or
|
||||||
|
requiring credentials or source data.
|
||||||
|
|
||||||
|
**Alternative:** Preserve offline checking of completed targets by pursuing a
|
||||||
|
dedicated upstream API that resolves targets without artifacts or credentials.
|
||||||
|
That would catch more missing endpoint/model settings before a run and could
|
||||||
|
serve other consumers, but it would add an upstream dependency to this work or
|
||||||
|
require a subsequent release. Reimplementing completion rules in Notarius
|
||||||
|
would risk a discrepancy with actual execution.
|
||||||
|
|
||||||
|
### Operator Diagnostics In This Upgrade
|
||||||
|
|
||||||
|
**Recommendation:** Migrate existing inspection commands and use native
|
||||||
|
inventory and observation APIs where they replace local work. Evaluate new
|
||||||
|
commands for provider smoke checks or effective-config diffs separately,
|
||||||
|
after identifying the operator workflows and output contracts they need.
|
||||||
|
|
||||||
|
**Alternative:** Include an explicitly invoked smoke command in the cutover.
|
||||||
|
This would help operators verify credentials and structured inference after
|
||||||
|
migrating a deployment, and it can delegate to `Engine.SmokeTest`. It adds a
|
||||||
|
network-capable CLI surface, with cost and output handling to define. Keep it
|
||||||
|
separate from ordinary offline validation in either case.
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
version: 5
|
version: 5
|
||||||
configkit:
|
config:
|
||||||
imports:
|
imports:
|
||||||
- path: pipelines/dnd-session.yml
|
- path: pipelines/dnd-session.yml
|
||||||
|
required: true
|
||||||
promptkit:
|
promptkit:
|
||||||
profile_file: ./llm-profiles/dnd-extraction.yml
|
profile_file: ./llm-profiles/dnd-extraction.yml
|
||||||
transient_retries: 2
|
transient_retries: 2
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
id: dnd-extraction
|
profiles:
|
||||||
backend: openrouter
|
dnd-extraction:
|
||||||
model: openai/gpt-5.6-luna
|
backend: openrouter
|
||||||
timeout_seconds: 240
|
model: openai/gpt-5.6-luna
|
||||||
service_tier: flex
|
timeout_seconds: 240
|
||||||
|
service_tier: flex
|
||||||
|
dnd-extraction-development:
|
||||||
|
extends: dnd-extraction
|
||||||
|
temperature: 0.2
|
||||||
|
|||||||
37
examples/pipelines/dnd-session.describe-session.yml
Normal file
37
examples/pipelines/dnd-session.describe-session.yml
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# Establish session-wide reference artifacts before their consumers.
|
||||||
|
id: describe-session
|
||||||
|
artifacts:
|
||||||
|
item-registry:
|
||||||
|
extract:
|
||||||
|
module: dnd/item-registry
|
||||||
|
retries: 2
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
normalize:
|
||||||
|
module: dnd/item-registry
|
||||||
|
npc-registry:
|
||||||
|
extract:
|
||||||
|
module: dnd/npc-registry
|
||||||
|
retries: 2
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
normalize:
|
||||||
|
module: dnd/npc-registry
|
||||||
|
retries: 2
|
||||||
|
location-registry:
|
||||||
|
extract:
|
||||||
|
module: dnd/location-registry
|
||||||
|
retries: 2
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
normalize:
|
||||||
|
module: dnd/location-registry
|
||||||
|
retries: 2
|
||||||
|
scene-descriptions:
|
||||||
|
extract:
|
||||||
|
module: dnd/scene-descriptions
|
||||||
|
retries: 2
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
normalize:
|
||||||
|
module: dnd/scene-descriptions
|
||||||
@@ -1,3 +1,9 @@
|
|||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: dnd-session.describe-session.yml
|
||||||
|
required: true
|
||||||
|
destination: [pipelines, dnd-session, steps]
|
||||||
|
placement: sequence-item
|
||||||
pipelines:
|
pipelines:
|
||||||
dnd-session:
|
dnd-session:
|
||||||
llm_profile: dnd-extraction
|
llm_profile: dnd-extraction
|
||||||
@@ -30,43 +36,6 @@ pipelines:
|
|||||||
- npc-occurrences
|
- npc-occurrences
|
||||||
- enemy-events
|
- enemy-events
|
||||||
steps:
|
steps:
|
||||||
# Establish session-wide reference artifacts before their consumers.
|
|
||||||
- id: describe-session
|
|
||||||
artifacts:
|
|
||||||
item-registry:
|
|
||||||
extract:
|
|
||||||
module: dnd/item-registry
|
|
||||||
retries: 2
|
|
||||||
merge:
|
|
||||||
module: appendorder
|
|
||||||
normalize:
|
|
||||||
module: dnd/item-registry
|
|
||||||
npc-registry:
|
|
||||||
extract:
|
|
||||||
module: dnd/npc-registry
|
|
||||||
retries: 2
|
|
||||||
merge:
|
|
||||||
module: appendorder
|
|
||||||
normalize:
|
|
||||||
module: dnd/npc-registry
|
|
||||||
retries: 2
|
|
||||||
location-registry:
|
|
||||||
extract:
|
|
||||||
module: dnd/location-registry
|
|
||||||
retries: 2
|
|
||||||
merge:
|
|
||||||
module: appendorder
|
|
||||||
normalize:
|
|
||||||
module: dnd/location-registry
|
|
||||||
retries: 2
|
|
||||||
scene-descriptions:
|
|
||||||
extract:
|
|
||||||
module: dnd/scene-descriptions
|
|
||||||
retries: 2
|
|
||||||
merge:
|
|
||||||
module: appendorder
|
|
||||||
normalize:
|
|
||||||
module: dnd/scene-descriptions
|
|
||||||
- id: extract-events
|
- id: extract-events
|
||||||
# Accepted registry artifacts and scene-description eligibility artifacts
|
# Accepted registry artifacts and scene-description eligibility artifacts
|
||||||
# are supplied in memory to their compatible consumers in this step.
|
# are supplied in memory to their compatible consumers in this step.
|
||||||
@@ -162,3 +131,9 @@ pipelines:
|
|||||||
module: appendorder
|
module: appendorder
|
||||||
normalize:
|
normalize:
|
||||||
module: dnd/enemy-events
|
module: dnd/enemy-events
|
||||||
|
|
||||||
|
# Development variant: disable PromptKit structural repairs while retaining
|
||||||
|
# the inherited steps and configured producer retry budgets.
|
||||||
|
dnd-session-development:
|
||||||
|
extends: dnd-session
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
|||||||
17
go.mod
17
go.mod
@@ -1,18 +1,17 @@
|
|||||||
module gitea.maximumdirect.net/eric/notarius
|
module gitea.maximumdirect.net/eric/notarius
|
||||||
|
|
||||||
go 1.25.5
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2
|
gitea.maximumdirect.net/eric/promptkit v0.12.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
go.maximumdirect.net/configkit v1.2.0
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||||
|
golang.org/x/text v0.40.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.40.0
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
20
go.sum
20
go.sum
@@ -1,20 +1,20 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2 h1:ZMxmv8pOnLtAx2hFTf8Z+pHKUbR89xErkBhwjr4yK24=
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2 h1:2RqI+KtflvSh7vLQaTAQCmMaFQscEA7EPwPazyvXRoY=
|
gitea.maximumdirect.net/eric/promptkit v0.12.0 h1:oIFWEg6GPxtmX3Plqv0euP+FnK1LCk/t4xDy6oKCpP4=
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2/go.mod h1:onm/FBPUKQ8/NSPwTEk26Ut+dVX2iXcq8vEseB+g/0E=
|
gitea.maximumdirect.net/eric/promptkit v0.12.0/go.mod h1:pFOOy5v1zH17stEi9rmHYmfbuLjnfVMcUhfmk+4VtJY=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
|
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8=
|
||||||
|
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -104,27 +104,6 @@ func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigDiscoveryUsesCompiledDefaultOnlyWhenAvailable(t *testing.T) {
|
|
||||||
info, statErr := os.Stat(defaultConfigPath)
|
|
||||||
if statErr != nil && !os.IsNotExist(statErr) {
|
|
||||||
t.Fatalf("stat compiled default config: %v", statErr)
|
|
||||||
}
|
|
||||||
if statErr == nil && !info.Mode().IsRegular() {
|
|
||||||
t.Skipf("compiled default config has unexpected host state: %s", info.Mode())
|
|
||||||
}
|
|
||||||
|
|
||||||
path, err := discoverConfigPath("", commandContractOptions(t))
|
|
||||||
if statErr == nil {
|
|
||||||
if err != nil || path != defaultConfigPath {
|
|
||||||
t.Fatalf("discoverConfigPath() = %q, %v; want compiled default", path, err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "config file not found") {
|
|
||||||
t.Fatalf("discoverConfigPath() error = %v, want documented not-found context", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
|
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
|
||||||
missing := filepath.Join(t.TempDir(), "missing.yml")
|
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|||||||
@@ -3,13 +3,15 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
|
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
@@ -17,6 +19,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
configPath := fs.String("config", "", "config file path")
|
configPath := fs.String("config", "", "config file path")
|
||||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
@@ -25,12 +28,28 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if *scope != "notarius" && *scope != "promptkit" {
|
||||||
|
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeConfigLoadError(stderr, err)
|
writeConfigLoadError(stderr, err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
if err := loaded.WriteEffective(stdout); err != nil {
|
if *scope == "notarius" {
|
||||||
|
if err := loaded.WriteEffective(stdout); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if err := inspector.WriteEffectiveConfiguration(stdout); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
@@ -42,6 +61,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
configPath := fs.String("config", "", "config file path")
|
configPath := fs.String("config", "", "config file path")
|
||||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
@@ -50,12 +70,42 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
|
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if *scope != "notarius" && *scope != "promptkit" {
|
||||||
|
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
pointer := fs.Arg(0)
|
pointer := fs.Arg(0)
|
||||||
parsed, err := configkit.ParsePath(pointer)
|
parsed, err := configkit.ParsePath(pointer)
|
||||||
if err != nil || parsed.String() != pointer {
|
if err != nil || parsed.String() != pointer {
|
||||||
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if *scope == "promptkit" {
|
||||||
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
|
if err != nil {
|
||||||
|
writeConfigLoadError(stderr, err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
found, err := inspector.WriteConfigurationExplanation(stdout, pointer)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, frameworkllm.ErrInvalidPromptKitConfigurationPointer) {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "notarius: write PromptKit configuration explanation: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
fmt.Fprintf(stderr, "notarius: PromptKit configuration pointer %q is unknown\n", pointer)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeConfigLoadError(stderr, err)
|
writeConfigLoadError(stderr, err)
|
||||||
@@ -68,6 +118,58 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runConfigProfile(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
|
fs := flag.NewFlagSet("config profile", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
configPath := fs.String("config", "", "config file path")
|
||||||
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() != 1 || strings.TrimSpace(fs.Arg(0)) == "" {
|
||||||
|
fmt.Fprintln(stderr, "notarius: config profile requires exactly one nonblank profile ID")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
profileID := fs.Arg(0)
|
||||||
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
|
if err != nil {
|
||||||
|
writeConfigLoadError(stderr, err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
profile, err := inspector.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
result := struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
EffectiveModelParams frameworkllm.PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||||
|
}{
|
||||||
|
ProfileID: profile.ProfileID,
|
||||||
|
APIKeyRequired: profile.CredentialRequired,
|
||||||
|
EffectiveModelParams: profile.EffectiveModelParams,
|
||||||
|
}
|
||||||
|
if err := json.NewEncoder(stdout).Encode(result); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: write PromptKit profile inspection: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
|
||||||
|
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
|
||||||
|
Source: promptKitProfileSourceConfig(cfg),
|
||||||
|
Assets: opts.promptKitAssets,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -14,13 +15,30 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) {
|
||||||
|
for _, scope := range []string{"notarius", "promptkit"} {
|
||||||
|
t.Run(scope, func(t *testing.T) {
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
options := commandContractOptions(t)
|
||||||
|
options.LookupEnv = func(string) (string, bool) {
|
||||||
|
t.Fatal("configuration discovery ran for invalid syntax")
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options)
|
||||||
|
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
path := filepath.Join(root, "notarius.yml")
|
path := filepath.Join(root, "notarius.yml")
|
||||||
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
||||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
||||||
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
|
writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n")
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
||||||
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
@@ -40,6 +58,130 @@ func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
path := writeCommandConfigContent(t, `version: 5
|
||||||
|
pipelines:
|
||||||
|
" parent ":
|
||||||
|
input:
|
||||||
|
module: seriatim
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract:
|
||||||
|
module: dnd/spells
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
child:
|
||||||
|
extends: " parent "
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
`)
|
||||||
|
options := commandContractOptions(t)
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing parent",
|
||||||
|
content: `version: 5
|
||||||
|
pipelines:
|
||||||
|
child:
|
||||||
|
extends: absent
|
||||||
|
`,
|
||||||
|
args: []string{"config", "validate", "--pipeline", "child"},
|
||||||
|
want: "inheritance.missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cycle",
|
||||||
|
content: `version: 5
|
||||||
|
pipelines:
|
||||||
|
first:
|
||||||
|
extends: second
|
||||||
|
second:
|
||||||
|
extends: first
|
||||||
|
`,
|
||||||
|
args: []string{"config", "validate", "--pipeline", "first"},
|
||||||
|
want: "cycle",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid directive type",
|
||||||
|
content: `version: 5
|
||||||
|
pipelines:
|
||||||
|
child:
|
||||||
|
extends: 7
|
||||||
|
`,
|
||||||
|
args: []string{"config", "validate", "--pipeline", "child"},
|
||||||
|
want: "extends",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unselected missing parent",
|
||||||
|
content: `version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input:
|
||||||
|
module: seriatim
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
unrelated:
|
||||||
|
extends: absent
|
||||||
|
`,
|
||||||
|
args: []string{"config", "validate", "--pipeline", "child"},
|
||||||
|
want: "inheritance.missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "exact sibling mismatch",
|
||||||
|
content: `version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input:
|
||||||
|
module: seriatim
|
||||||
|
child:
|
||||||
|
extends: "parent "
|
||||||
|
`,
|
||||||
|
args: []string{"pipelines", "list"},
|
||||||
|
want: "inheritance.missing",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
path := writeCommandConfigContent(t, test.content)
|
||||||
|
args := append([]string{}, test.args...)
|
||||||
|
args = append(args, "--config", path)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -152,6 +294,229 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
root := t.TempDir()
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
||||||
|
base:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: base-model
|
||||||
|
extra_params:
|
||||||
|
Authorization: base-auth-secret
|
||||||
|
ordinary: visible-value
|
||||||
|
`)
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
||||||
|
imports:
|
||||||
|
- path: base.yml
|
||||||
|
profiles:
|
||||||
|
"escaped/id":
|
||||||
|
model: imported-model
|
||||||
|
dnd-extraction:
|
||||||
|
model: imported-override
|
||||||
|
`)
|
||||||
|
assets, err := productionPromptAssets()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
factoryCalls := 0
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
opts.promptKitAssets = assets
|
||||||
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return nil, nil, errors.New("inspection must not construct an LLM client")
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
defaultReport := stdout.String()
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if factoryCalls != 0 {
|
||||||
|
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
path := writeCommandConfig(t, "session", "other")
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
||||||
|
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
||||||
|
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
||||||
|
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
||||||
|
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
root := t.TempDir()
|
||||||
|
profilePath := filepath.Join(root, "profiles.yml")
|
||||||
|
writeConfigKitFile(t, profilePath, `backends:
|
||||||
|
managed:
|
||||||
|
endpoint: https://provider.example.invalid/v1
|
||||||
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||||
|
profiles:
|
||||||
|
inherited:
|
||||||
|
backend: managed
|
||||||
|
model: inherited-model
|
||||||
|
temperature: 0.4
|
||||||
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||||
|
api_key_required: true
|
||||||
|
extra_params:
|
||||||
|
Authorization: private-authorization
|
||||||
|
ordinary: visible-value
|
||||||
|
selected:
|
||||||
|
extends: inherited
|
||||||
|
temperature: 0
|
||||||
|
extra_params: null
|
||||||
|
endpoint-only:
|
||||||
|
endpoint: https://endpoint.example.invalid/v1
|
||||||
|
model: endpoint-model
|
||||||
|
api_key_required: false
|
||||||
|
" literal profile ":
|
||||||
|
endpoint: https://literal.example.invalid/v1
|
||||||
|
model: literal-model
|
||||||
|
`)
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
assets, err := productionPromptAssets()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lookupCalls := 0
|
||||||
|
factoryCalls := 0
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
opts.promptKitAssets = assets
|
||||||
|
opts.LookupEnv = func(string) (string, bool) {
|
||||||
|
lookupCalls++
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return nil, nil, errors.New("profile inspection must not construct an LLM client")
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileReport struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
Target struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
} `json:"effective_model_params"`
|
||||||
|
}
|
||||||
|
run := func(profileID string) (int, string, string, profileReport) {
|
||||||
|
t.Helper()
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
|
||||||
|
var report profileReport
|
||||||
|
if code == 0 {
|
||||||
|
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
|
||||||
|
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||||
|
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return code, stdout.String(), stderr.String(), report
|
||||||
|
}
|
||||||
|
code, output, diagnostic, selected := run("selected")
|
||||||
|
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
|
||||||
|
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, inherited := run("inherited")
|
||||||
|
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
|
||||||
|
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, endpointOnly := run("endpoint-only")
|
||||||
|
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
|
||||||
|
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, literal := run(" literal profile ")
|
||||||
|
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
|
||||||
|
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
|
||||||
|
}
|
||||||
|
if lookupCalls != 0 || factoryCalls != 0 {
|
||||||
|
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
configPath := writeCommandConfig(t, "session", "other")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
|
||||||
|
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
|
||||||
|
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
|
||||||
|
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||||
|
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
|
||||||
|
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
|
||||||
|
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{}
|
type failingInspectionWriter struct{}
|
||||||
|
|
||||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||||
|
|||||||
@@ -5,25 +5,32 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func loadConfigKit(configPath, overlay string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
|
func loadConfigKit(configPath, overlay string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
|
||||||
path, err := discoverConfigPath(configPath, opts)
|
var source configkit.Source
|
||||||
if err != nil {
|
if path := strings.TrimSpace(configPath); path != "" {
|
||||||
return nil, "", err
|
source = configkit.File(path)
|
||||||
|
} else if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
|
||||||
|
source = configkit.File(strings.TrimSpace(path))
|
||||||
|
} else {
|
||||||
|
source = configkit.SearchConfig("notarius", configkit.Searches(
|
||||||
|
configkit.WorkingDirectory(), configkit.UserConfigDirectory(), configkit.UnixSystemDirectories(),
|
||||||
|
), configkit.SearchOptions{Required: true})
|
||||||
}
|
}
|
||||||
workingDirectory, err := os.Getwd()
|
workingDirectory, err := os.Getwd()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("determine invocation directory: %w", err)
|
return nil, "", fmt.Errorf("determine invocation directory: %w", err)
|
||||||
}
|
}
|
||||||
loaded, err := config.Load(path, config.LoadOptions{Overlay: overlay, Overrides: overrides, InvocationDirectory: workingDirectory})
|
loaded, err := config.LoadSource(source, config.LoadOptions{Overlay: overlay, Overrides: overrides, InvocationDirectory: workingDirectory})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
return loaded, path, nil
|
return loaded, loaded.MainSourcePath(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeConfigLoadError(output io.Writer, err error) {
|
func writeConfigLoadError(output io.Writer, err error) {
|
||||||
|
|||||||
@@ -4,11 +4,12 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
|
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
|
||||||
@@ -24,7 +25,7 @@ func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
|
|||||||
writeConfigKitFile(t, main, "version: 5\n")
|
writeConfigKitFile(t, main, "version: 5\n")
|
||||||
args = append(args, "--config-overlay", "development")
|
args = append(args, "--config-overlay", "development")
|
||||||
} else {
|
} else {
|
||||||
writeConfigKitFile(t, main, "version: 5\nconfigkit:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n")
|
writeConfigKitFile(t, main, "version: 5\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n")
|
||||||
}
|
}
|
||||||
writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n")
|
writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n")
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -59,6 +60,79 @@ func TestLoadConfigKitUsesDiscoveryOverlayAndOverrides(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNativeDiscoverySelectionAndTerminalFailures(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
work := filepath.Join(root, "work")
|
||||||
|
user := filepath.Join(root, "user")
|
||||||
|
if err := os.MkdirAll(work, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "darwin":
|
||||||
|
t.Setenv("HOME", user)
|
||||||
|
user = filepath.Join(user, "Library", "Application Support")
|
||||||
|
case "windows":
|
||||||
|
t.Setenv("AppData", user)
|
||||||
|
default:
|
||||||
|
t.Setenv("XDG_CONFIG_HOME", user)
|
||||||
|
}
|
||||||
|
previous, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(work); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := os.Chdir(previous); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
userFile := filepath.Join(user, "notarius", "config.yml")
|
||||||
|
writeConfigKitFile(t, userFile, "version: 5\npipelines: {user: {input: {module: input}}}\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 5\npipelines: {generic: {input: {module: input}}}\n")
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
loaded, path, err := loadConfigKit("", "", nil, opts)
|
||||||
|
if err != nil || path != userFile || loaded.Config.Pipelines["user"].Input.Module != "input" {
|
||||||
|
t.Fatalf("user discovery = %q, %v", path, err)
|
||||||
|
}
|
||||||
|
workingYAML := filepath.Join(work, "notarius.yaml")
|
||||||
|
workingYML := filepath.Join(work, "notarius.yml")
|
||||||
|
writeConfigKitFile(t, workingYAML, "version: 5\npipelines: {yaml: {input: {module: input}}}\n")
|
||||||
|
writeConfigKitFile(t, workingYML, "version: 5\npipelines: {yml: {input: {module: input}}}\n")
|
||||||
|
loaded, path, err = loadConfigKit("", "", nil, opts)
|
||||||
|
if err != nil || path != workingYML || loaded.Config.Pipelines["yml"].Input.Module != "input" || loaded.Config.Output.Directory != filepath.Join(work, "notarius-output") {
|
||||||
|
t.Fatalf("working discovery = %q, %v", path, err)
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"pipelines", "list"}, {"config", "validate"}, {"config", "effective"}, {"config", "explain", "/version"}} {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 || stdout.Len() == 0 {
|
||||||
|
t.Fatalf("%v: code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
missingEnv := opts
|
||||||
|
missingEnv.LookupEnv = func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" }
|
||||||
|
if _, _, err := loadConfigKit("", "", nil, missingEnv); err == nil {
|
||||||
|
t.Fatal("missing environment selection fell back")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(workingYML, []byte("version: [\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := loadConfigKit("", "", nil, opts); err == nil {
|
||||||
|
t.Fatal("invalid selected file fell back")
|
||||||
|
}
|
||||||
|
if _, _, err := loadConfigKit(filepath.Join(root, "missing.yml"), "", nil, opts); err == nil {
|
||||||
|
t.Fatal("missing explicit file fell back")
|
||||||
|
}
|
||||||
|
if err := os.Remove(workingYML); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, path, err = loadConfigKit("", "", nil, opts)
|
||||||
|
if err != nil || path != workingYAML || loaded.Config.Pipelines["yaml"].Input.Module != "input" {
|
||||||
|
t.Fatalf("alternate named file = %q, %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) {
|
func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
path := filepath.Join(t.TempDir(), "invalid.yml")
|
path := filepath.Join(t.TempDir(), "invalid.yml")
|
||||||
|
|||||||
@@ -286,6 +286,13 @@ func completeExampleConfigWithTemporaryCache(t *testing.T) string {
|
|||||||
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.yml"), fragment, 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.yml"), fragment, 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
description, err := os.ReadFile(repositoryPath("examples", "pipelines", "dnd-session.describe-session.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "pipelines", "dnd-session.describe-session.yml"), description, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
path := filepath.Join(root, "dnd-complete.config.yml")
|
path := filepath.Join(root, "dnd-complete.config.yml")
|
||||||
if err := os.WriteFile(path, []byte(updated), 0o600); err != nil {
|
if err := os.WriteFile(path, []byte(updated), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -25,6 +26,7 @@ import (
|
|||||||
|
|
||||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
|
completePipelines := make(map[string]pipeline.ResolvedPipeline)
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
t.Run(example.name, func(t *testing.T) {
|
t.Run(example.name, func(t *testing.T) {
|
||||||
cfg := loadMaintainedExample(t, example.path)
|
cfg := loadMaintainedExample(t, example.path)
|
||||||
@@ -58,9 +60,45 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|||||||
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
||||||
}
|
}
|
||||||
if example.name == "complete" {
|
if example.name == "complete" {
|
||||||
|
if pipelineID == "dnd-session-development" {
|
||||||
|
parent, found := completePipelines["dnd-session"]
|
||||||
|
if !found {
|
||||||
|
t.Fatal("complete parent pipeline was not resolved before its development variant")
|
||||||
|
}
|
||||||
|
if strings.Join(exampleStepLaneIDs(materialized), "|") != strings.Join(exampleStepLaneIDs(parent), "|") {
|
||||||
|
t.Fatalf("development variant steps and lanes = %v, want parent composition %v", exampleStepLaneIDs(materialized), exampleStepLaneIDs(parent))
|
||||||
|
}
|
||||||
|
if got, want := strings.Join(generatedReferenceBindingDescriptions(materialized), "|"), strings.Join(generatedReferenceBindingDescriptions(parent), "|"); got != want {
|
||||||
|
t.Fatalf("development generated reference bindings = %q, want parent bindings %q", got, want)
|
||||||
|
}
|
||||||
|
eligibleBindings := 0
|
||||||
|
for _, step := range materialized.Steps {
|
||||||
|
for _, lane := range step.ArtifactLanes {
|
||||||
|
for _, binding := range []struct {
|
||||||
|
module pipeline.ModuleBinding
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
|
}{
|
||||||
|
{module: lane.Extract, executionClass: lane.ExtractExecutionClass},
|
||||||
|
{module: lane.Normalize, executionClass: lane.NormalizeExecutionClass},
|
||||||
|
} {
|
||||||
|
if binding.executionClass != "llm_backed" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
eligibleBindings++
|
||||||
|
if got := binding.module.StructuredOutputRepairAttempts; got == nil || *got != 0 {
|
||||||
|
t.Fatalf("development %s repair attempts = %v, want explicit zero", binding.module.Module, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if eligibleBindings == 0 {
|
||||||
|
t.Fatal("development variant had no eligible LLM bindings for repair policy")
|
||||||
|
}
|
||||||
|
}
|
||||||
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-registry,location-registry,npc-registry,scene-descriptions|extract-events:combat-turns,item-occurrences,location-occurrences,npc-occurrences,spells|track-enemies:enemy-events" {
|
||||||
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
t.Fatalf("complete example steps and lanes = %v, want the documented D&D extractor composition", got)
|
||||||
}
|
}
|
||||||
|
completePipelines[pipelineID] = materialized
|
||||||
locationLane := referenceContractLane(t, materialized, "location-registry")
|
locationLane := referenceContractLane(t, materialized, "location-registry")
|
||||||
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
if locationLane.ArtifactKind != dnd.LocationRegistryKind || locationLane.Extract.Module != locationextract.Key || locationLane.Extract.Retries != 2 || locationLane.Merge.Module != pipeline.DefaultMergeModule || locationLane.Normalize.Module != locationnormalize.Key || locationLane.Normalize.Retries != 2 {
|
||||||
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
t.Fatalf("location lane = %#v, want typed registry composition", locationLane)
|
||||||
@@ -155,7 +193,7 @@ func TestMaintainedConfigurationExampleSet(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
|
if len(llmProfiles) != 1 || llmProfiles[0].Name() != "dnd-extraction.yml" {
|
||||||
t.Fatalf("maintained LLM profiles = %#v, want dnd-extraction.yml", llmProfiles)
|
t.Fatalf("maintained LLM profile files = %#v, want the single combined profile file", llmProfiles)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,15 +202,38 @@ func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
|||||||
t.Setenv("OPENROUTER_API_KEY", "")
|
t.Setenv("OPENROUTER_API_KEY", "")
|
||||||
for _, example := range maintainedExampleFiles(t) {
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
t.Run(example.name, func(t *testing.T) {
|
t.Run(example.name, func(t *testing.T) {
|
||||||
var stdout, stderr strings.Builder
|
for _, pipelineID := range example.pipelineIDs {
|
||||||
code := RunWithOptions([]string{
|
t.Run(pipelineID, func(t *testing.T) {
|
||||||
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
|
var stdout, stderr strings.Builder
|
||||||
}, &stdout, &stderr, Options{})
|
code := RunWithOptions([]string{
|
||||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
|
"config", "validate", "--config", example.path, "--pipeline", pipelineID,
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
}, &stdout, &stderr, Options{})
|
||||||
|
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "`+pipelineID+`"`) {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"config", "profile", "--config", repositoryPath("examples", "dnd-complete.config.yml"), "dnd-extraction-development",
|
||||||
|
}, &stdout, &stderr, Options{})
|
||||||
|
var profile struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
Target struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
} `json:"effective_model_params"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(stdout.String()), &profile); err != nil {
|
||||||
|
t.Fatalf("development profile JSON = %q: %v", stdout.String(), err)
|
||||||
|
}
|
||||||
|
if code != 0 || stderr.Len() != 0 || profile.ProfileID != "dnd-extraction-development" || profile.Target.BackendID != "openrouter" || profile.Target.Model != "openai/gpt-5.6-luna" || profile.Target.Temperature != 0.2 {
|
||||||
|
t.Fatalf("development profile inspection = code %d, %q, %q, %#v", code, stdout.String(), stderr.String(), profile)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||||
@@ -188,6 +249,31 @@ func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func generatedReferenceBindingDescriptions(resolved pipeline.ResolvedPipeline) []string {
|
||||||
|
var descriptions []string
|
||||||
|
for _, step := range resolved.Steps {
|
||||||
|
for _, lane := range step.ArtifactLanes {
|
||||||
|
for _, target := range []struct {
|
||||||
|
name string
|
||||||
|
target pipeline.ResolvedReferenceTarget
|
||||||
|
}{
|
||||||
|
{name: "extract", target: lane.ExtractReferences},
|
||||||
|
{name: "merge", target: lane.MergeReferences},
|
||||||
|
{name: "normalize", target: lane.NormalizeReferences},
|
||||||
|
} {
|
||||||
|
for _, binding := range target.target.Bindings {
|
||||||
|
if binding.Artifact == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
descriptions = append(descriptions, fmt.Sprintf("%s/%s/%s/%s=%s/%s", step.ID, lane.ID, target.name, binding.SlotName, binding.Artifact.Step, binding.Artifact.Lane))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(descriptions)
|
||||||
|
return descriptions
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
fake := &productionFakeLLMClient{}
|
fake := &productionFakeLLMClient{}
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||||
@@ -59,6 +60,7 @@ import (
|
|||||||
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -228,14 +230,16 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
|||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
newEngine := func(profileFile string) (*promptkit.Engine, error) {
|
newEngine := func(profileFile string) (*promptkit.Engine, error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
options, err := components.assets.PromptKitOptions()
|
sources, options, err := components.assets.PromptKitConfigurationSources()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if profileFile != "" {
|
if profileFile != "" {
|
||||||
options = append(options, promptkit.WithProfileFile(profileFile))
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
|
Source: configkit.File(profileFile), Imports: true,
|
||||||
|
}}})
|
||||||
}
|
}
|
||||||
return promptkit.NewEngine(promptkit.Config{}, options...)
|
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("fallback", func(t *testing.T) {
|
t.Run("fallback", func(t *testing.T) {
|
||||||
@@ -258,10 +262,11 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("valid operator profile wins", func(t *testing.T) {
|
t.Run("valid operator profile wins", func(t *testing.T) {
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
|
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||||
endpoint: http://operator.example.test/v1
|
dnd-extraction:
|
||||||
model: operator-model
|
endpoint: http://operator.example.test/v1
|
||||||
timeout_seconds: 75
|
model: operator-model
|
||||||
|
timeout_seconds: 75
|
||||||
`), 0o600); err != nil {
|
`), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -274,14 +279,14 @@ timeout_seconds: 75
|
|||||||
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
|
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
|
||||||
}
|
}
|
||||||
params := inspection.EffectiveModelParams
|
params := inspection.EffectiveModelParams
|
||||||
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
|
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
|
||||||
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
|
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
|
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
if err := os.WriteFile(profilePath, []byte("id: dnd-extraction\nendpoint: http://operator.example.test/v1\nmodel: operator-model\nunknown: value\n"), 0o600); err != nil {
|
if err := os.WriteFile(profilePath, []byte("profiles:\n dnd-extraction:\n endpoint: http://operator.example.test/v1\n model: operator-model\n unknown: value\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
engine, err := newEngine(profilePath)
|
engine, err := newEngine(profilePath)
|
||||||
@@ -320,17 +325,9 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
|||||||
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
||||||
t.Fatalf("generic reconciliation schema asset: %v", err)
|
t.Fatalf("generic reconciliation schema asset: %v", err)
|
||||||
}
|
}
|
||||||
options, err := components.assets.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||||
t.Fatalf("production PromptKit options: %v", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("production prompt engine: %v", err)
|
|
||||||
}
|
|
||||||
if engine == nil {
|
if engine == nil {
|
||||||
t.Fatal("production PromptKit engine is nil")
|
t.Fatal("production PromptKit engine is nil")
|
||||||
}
|
}
|
||||||
@@ -504,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
|
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
|
||||||
}
|
}
|
||||||
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
||||||
if err != nil || len(fingerprints) != 1 {
|
if err != nil || len(fingerprints) != 1 {
|
||||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
|
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
|
||||||
}
|
}
|
||||||
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
||||||
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
||||||
@@ -550,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
||||||
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -575,25 +572,27 @@ func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||||
if err := os.WriteFile(profilePath, []byte(`id: local-profile
|
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||||
backend: local
|
local-profile:
|
||||||
model: local-model
|
backend: local
|
||||||
|
model: local-model
|
||||||
`), 0o600); err != nil {
|
`), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
assets := llm.NewAssetRegistry()
|
assets := llm.NewAssetRegistry()
|
||||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||||
"production.local.yaml": {Data: []byte(`id: production.local
|
"production.local.yaml": {Data: []byte(`prompts:
|
||||||
version: "v1"
|
production.local:
|
||||||
inputs:
|
v1:
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
messages:
|
required: true
|
||||||
- role: user
|
messages:
|
||||||
content: '{{ input "transcript" }}'
|
- role: user
|
||||||
output:
|
content: '{{ input "transcript" }}'
|
||||||
format: json
|
output:
|
||||||
validation_mode: json
|
format: json
|
||||||
|
validation_mode: json
|
||||||
`)},
|
`)},
|
||||||
}, "."); err != nil {
|
}, "."); err != nil {
|
||||||
t.Fatalf("register prompt assets: %v", err)
|
t.Fatalf("register prompt assets: %v", err)
|
||||||
@@ -824,7 +823,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
return []maintainedExample{
|
return []maintainedExample{
|
||||||
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
||||||
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session"}},
|
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session", "dnd-session-development"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -31,11 +31,12 @@ func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *
|
|||||||
}
|
}
|
||||||
return profilePath
|
return profilePath
|
||||||
}
|
}
|
||||||
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
|
localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n"
|
||||||
credentialProfile := `id: credential-profile
|
credentialProfile := `profiles:
|
||||||
endpoint: ` + server.URL + `/v1
|
credential-profile:
|
||||||
model: credential-model
|
endpoint: ` + server.URL + `/v1
|
||||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
model: credential-model
|
||||||
|
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||||
`
|
`
|
||||||
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
|
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
|
||||||
|
|
||||||
@@ -60,7 +61,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
name: "missing local backend registration",
|
name: "missing local backend registration",
|
||||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||||
profileID: "local-profile",
|
profileID: "local-profile",
|
||||||
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
|
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "absent profile",
|
name: "absent profile",
|
||||||
@@ -71,9 +72,9 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "malformed profile",
|
name: "malformed profile",
|
||||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"),
|
||||||
profileID: "malformed-profile",
|
profileID: "malformed-profile",
|
||||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||||
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -149,7 +150,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
|||||||
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||||
assets := llm.NewAssetRegistry()
|
assets := llm.NewAssetRegistry()
|
||||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
"profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
|
||||||
}, "profiles"); err != nil {
|
}, "profiles"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -160,8 +161,8 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
|||||||
|
|
||||||
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||||
for _, profiles := range []string{
|
for _, profiles := range []string{
|
||||||
"id: child\nbase_profile: missing\n",
|
"profiles:\n child:\n extends: missing\n",
|
||||||
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
"profiles:\n first:\n extends: second\n second:\n extends: first\n",
|
||||||
} {
|
} {
|
||||||
t.Run("invalid inheritance", func(t *testing.T) {
|
t.Run("invalid inheritance", func(t *testing.T) {
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
@@ -169,7 +170,7 @@ func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGenera
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profileID := "child"
|
profileID := "child"
|
||||||
if strings.Contains(profiles, "id: first") {
|
if strings.Contains(profiles, "first:") {
|
||||||
profileID = "first"
|
profileID = "first"
|
||||||
}
|
}
|
||||||
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
@@ -27,16 +26,17 @@ import (
|
|||||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
notarius --version
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius config effective --config path/to/config.yml [--config-overlay name]
|
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
|
||||||
notarius config explain --config path/to/config.yml [--config-overlay name] <json-pointer>
|
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
||||||
`
|
`
|
||||||
|
|
||||||
@@ -1059,6 +1059,8 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
|
|||||||
return runConfigEffective(args[1:], stdout, stderr, opts)
|
return runConfigEffective(args[1:], stdout, stderr, opts)
|
||||||
case "explain":
|
case "explain":
|
||||||
return runConfigExplain(args[1:], stdout, stderr, opts)
|
return runConfigExplain(args[1:], stdout, stderr, opts)
|
||||||
|
case "profile":
|
||||||
|
return runConfigProfile(args[1:], stdout, stderr, opts)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
||||||
writeUsage(stderr)
|
writeUsage(stderr)
|
||||||
@@ -1082,39 +1084,6 @@ func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func discoverConfigPath(configPath string, opts Options) (string, error) {
|
|
||||||
if path := strings.TrimSpace(configPath); path != "" {
|
|
||||||
if err := requireConfigFile(path); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
|
|
||||||
path = strings.TrimSpace(path)
|
|
||||||
if err := requireConfigFile(path); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(defaultConfigPath); err == nil {
|
|
||||||
return defaultConfigPath, nil
|
|
||||||
} else if err != nil && !os.IsNotExist(err) {
|
|
||||||
return "", fmt.Errorf("check default config %q: %w", defaultConfigPath, err)
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf("config file not found; pass --config or set NOTARIUS_CONFIG")
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireConfigFile(path string) error {
|
|
||||||
info, err := os.Stat(path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("config file %q is not available: %w", path, err)
|
|
||||||
}
|
|
||||||
if info.IsDir() {
|
|
||||||
return fmt.Errorf("config file %q is a directory", path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOnly(raw string) ([]string, error) {
|
func parseOnly(raw string) ([]string, error) {
|
||||||
if strings.TrimSpace(raw) == "" {
|
if strings.TrimSpace(raw) == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -271,6 +271,41 @@ func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsDiscoveredConfigurationPath(t *testing.T) {
|
||||||
|
roots := newStateTestRoots(t)
|
||||||
|
work := t.TempDir()
|
||||||
|
selected := filepath.Join(work, "notarius.yml")
|
||||||
|
data, err := os.ReadFile(roots.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(selected, data, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previous, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(work); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := os.Chdir(previous); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"run", "sample", "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, newStateTestHarness().options())
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
var invocation debugbundle.Invocation
|
||||||
|
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "invocation.json", &invocation)
|
||||||
|
if invocation.ConfigPath != selected {
|
||||||
|
t.Fatalf("recorded config path = %q, want %q", invocation.ConfigPath, selected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
|
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
|
||||||
for _, overlay := range []string{"", "default"} {
|
for _, overlay := range []string{"", "default"} {
|
||||||
if got := effectiveConfigOverlay(overlay); got != "" {
|
if got := effectiveConfigOverlay(overlay); got != "" {
|
||||||
@@ -679,7 +714,7 @@ func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
|
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
|
||||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
fingerprints := prepared.CheckpointFingerprints()
|
fingerprints := prepared.CheckpointFingerprints()
|
||||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
|
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
|
||||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
|
|
||||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
|
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
|
||||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||||
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
|
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||||
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
|
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) {
|
|||||||
root := filepath.Join(t.TempDir(), "debug")
|
root := filepath.Join(t.TempDir(), "debug")
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options())
|
code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options())
|
||||||
if code != 1 || !strings.Contains(stderr.String(), "config file") {
|
if code != 1 || !strings.Contains(stderr.String(), "source.missing") {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
assertAbsent(t, root)
|
assertAbsent(t, root)
|
||||||
|
|||||||
305
internal/core/config/collection_composition_test.go
Normal file
305
internal/core/config/collection_composition_test.go
Normal file
@@ -0,0 +1,305 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
write := func(name, data string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(root, name)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write("notarius.yml", `version: 5
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: one/pipeline.yml
|
||||||
|
required: true
|
||||||
|
- path: two/pipeline.yml
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
write("one/pipeline.yml", `pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
references: {source: {path: ./first.txt}}
|
||||||
|
artifacts: {first: {extract: {module: extract, retries: 4}}}
|
||||||
|
`)
|
||||||
|
write("two/pipeline.yml", `pipelines:
|
||||||
|
session:
|
||||||
|
steps:
|
||||||
|
- id: second
|
||||||
|
references: {source: {path: ./second.txt}}
|
||||||
|
artifacts: {second: {extract: {module: extract}}}
|
||||||
|
`)
|
||||||
|
write("overlays/upsert.yml", `pipelines:
|
||||||
|
session:
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
artifacts: {replacement: {extract: {module: extract}}}
|
||||||
|
`)
|
||||||
|
write("overlays/replace.yml", `config:
|
||||||
|
sequences:
|
||||||
|
- destination: [pipelines, session, steps]
|
||||||
|
mode: replace
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
steps:
|
||||||
|
- id: third
|
||||||
|
artifacts: {third: {extract: {module: extract}}}
|
||||||
|
`)
|
||||||
|
write("overlays/append.yml", `config:
|
||||||
|
sequences:
|
||||||
|
- destination: [pipelines, session, steps]
|
||||||
|
mode: append
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
steps:
|
||||||
|
- id: third
|
||||||
|
artifacts: {third: {extract: {module: extract}}}
|
||||||
|
`)
|
||||||
|
write("overlays/clear.yml", `config:
|
||||||
|
sequences:
|
||||||
|
- destination: [pipelines, session, steps]
|
||||||
|
mode: replace
|
||||||
|
pipelines:
|
||||||
|
session: {steps: []}
|
||||||
|
`)
|
||||||
|
main := filepath.Join(root, "notarius.yml")
|
||||||
|
loaded, err := Load(main, LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps := loaded.Config.Pipelines["session"].Steps
|
||||||
|
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") {
|
||||||
|
t.Fatalf("composed steps = %#v", steps)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
overlay string
|
||||||
|
ids []string
|
||||||
|
}{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} {
|
||||||
|
t.Run(tc.overlay, func(t *testing.T) {
|
||||||
|
got, err := Load(main, LoadOptions{Overlay: tc.overlay})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps := got.Config.Pipelines["session"].Steps
|
||||||
|
if len(steps) != len(tc.ids) {
|
||||||
|
t.Fatalf("steps = %#v", steps)
|
||||||
|
}
|
||||||
|
for i, id := range tc.ids {
|
||||||
|
if steps[i].ID != id {
|
||||||
|
t.Fatalf("steps = %#v", steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") {
|
||||||
|
t.Fatalf("upsert did not replace record: %#v", steps[0])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil {
|
||||||
|
t.Fatal("domain validation accepted empty steps")
|
||||||
|
}
|
||||||
|
overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}}
|
||||||
|
replaced, err := Load(main, LoadOptions{Overrides: overrides})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" {
|
||||||
|
t.Fatalf("override steps = %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, first, second string
|
||||||
|
}{
|
||||||
|
{"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"},
|
||||||
|
{"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, data := range map[string]string{
|
||||||
|
"main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n",
|
||||||
|
"one.yml": "pipelines:\n session:\n " + tc.first + "\n",
|
||||||
|
"two.yml": "pipelines:\n session:\n " + tc.second + "\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil {
|
||||||
|
t.Fatal("conflicting peer lists were accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
main := filepath.Join(root, "main.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
artifacts: {old: {extract: {module: extract, retries: 4}}}
|
||||||
|
- id: second
|
||||||
|
artifacts: {second: {extract: {module: extract}}}
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
artifacts: {new: {extract: {module: extract}}}
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := Load(main, LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps := loaded.Config.Pipelines["child"].Steps
|
||||||
|
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" {
|
||||||
|
t.Fatalf("inherited steps = %#v", steps)
|
||||||
|
}
|
||||||
|
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil {
|
||||||
|
t.Fatalf("resolve composed pipeline: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentifiedStepInheritanceDoesNotCreateTemplateSteps(t *testing.T) {
|
||||||
|
main := filepath.Join(t.TempDir(), "main.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
steps:
|
||||||
|
- id: parent
|
||||||
|
references: {source: {path: ./shared.txt}}
|
||||||
|
- id: child
|
||||||
|
extends: {id: parent}
|
||||||
|
artifacts: {result: {extract: {module: extract}}}
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := Load(main, LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps := loaded.Config.Pipelines["session"].Steps
|
||||||
|
if len(steps) != 2 || steps[1].ID != "child" || steps[1].References["source"].Path != filepath.Join(filepath.Dir(main), "shared.txt") {
|
||||||
|
t.Fatalf("inherited step = %#v", steps)
|
||||||
|
}
|
||||||
|
if err := loaded.Config.Validate(); err != nil {
|
||||||
|
t.Fatalf("validate inherited steps: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}); err == nil || !strings.Contains(err.Error(), `step "parent" must declare at least one artifact lane`) {
|
||||||
|
t.Fatalf("expected non-executable parent rejection, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentifiedValidatorRecordInheritanceResolves(t *testing.T) {
|
||||||
|
main := filepath.Join(t.TempDir(), "main.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
config:
|
||||||
|
sequences:
|
||||||
|
- destination: [pipelines, session, artifacts, result, extract, validators]
|
||||||
|
key: module
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
artifacts:
|
||||||
|
result:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
validators:
|
||||||
|
- module: configured-a
|
||||||
|
options: {nested: {inherited: true}}
|
||||||
|
- module: configured-b
|
||||||
|
extends: {module: configured-a}
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := Load(main, LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
effective, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, chain := range effective.ResolvedPipeline.ValidatorChains {
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
if validator.Binding.Module == "configured-b" {
|
||||||
|
if nested, ok := validator.Binding.Options["nested"].(map[string]any); !ok || nested["inherited"] != true {
|
||||||
|
t.Fatalf("inherited options = %#v", validator.Binding.Options)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("inheriting validator missing from resolved pipeline")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyStepOperationsAndOptionalImports(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
main := filepath.Join(root, "main.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: absent.yml
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
artifacts: {result: {extract: {module: extract}}}
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
mode, value string
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
{"append", "[]", false}, {"upsert", "[]", false}, {"replace", "[]", true}, {"replace", "null", true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.mode+tc.value, func(t *testing.T) {
|
||||||
|
name := tc.mode + map[string]string{"[]": "empty", "null": "null"}[tc.value]
|
||||||
|
path := filepath.Join(root, "overlays", name+".yml")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data := "config:\n sequences:\n - destination: [pipelines, session, steps]\n mode: " + tc.mode + "\npipelines:\n session: {steps: " + tc.value + "}\n"
|
||||||
|
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := Load(main, LoadOptions{Overlay: name})
|
||||||
|
if tc.wantError {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("invalid steps were accepted")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
steps := loaded.Config.Pipelines["session"].Steps
|
||||||
|
if len(steps) != 1 || steps[0].ID != "first" {
|
||||||
|
t.Fatalf("steps = %#v", steps)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,26 +1,16 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
import configkit "go.maximumdirect.net/configkit"
|
||||||
|
|
||||||
// documentNormalizer records leaf provenance and, while visiting a sequence,
|
// A step whose identity is normalized no longer corresponds to its original
|
||||||
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
|
// record. Its ancestor record covers its edits instead of invalid leaf records.
|
||||||
// values, so a record for only a descendant is insufficient. The outermost
|
|
||||||
// sequence covers nested sequences as well; unchanged visits record nothing.
|
|
||||||
type documentNormalizer struct {
|
type documentNormalizer struct {
|
||||||
*configkit.Resolution
|
*configkit.Resolution
|
||||||
sequence *configkit.Path
|
covered bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
|
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
|
||||||
if n.sequence == nil {
|
if !n.covered {
|
||||||
n.sequence = &path
|
n.Resolution.RecordNormalization(path, reason, dependencies...)
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n documentNormalizer) Record(path configkit.Path, kind configkit.ChangeKind, reason string, dependencies ...configkit.Path) {
|
|
||||||
n.Resolution.Record(path, kind, reason, dependencies...)
|
|
||||||
if n.sequence != nil {
|
|
||||||
n.Resolution.Record(*n.sequence, kind, reason, append([]configkit.Path{path}, dependencies...)...)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,12 +2,13 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
|
func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
|
||||||
resolution := documentNormalizer{Resolution: context}
|
normalizer := documentNormalizer{Resolution: resolution}
|
||||||
if document == nil {
|
if document == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -33,25 +34,29 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
|||||||
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
||||||
profile := document.Pipelines[id]
|
profile := document.Pipelines[id]
|
||||||
path := configkit.Keys("pipelines").Key(id)
|
path := configkit.Keys("pipelines").Key(id)
|
||||||
normalizeDocumentReferencePaths(profile.References, path.Key("references"), resolution, state)
|
normalizeDocumentReferencePaths(profile.References, path.Key("references"), normalizer, state)
|
||||||
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), resolution, state)
|
normalizeDocumentBindingReferencePaths(&profile.Input, path.Key("input"), normalizer, state)
|
||||||
if profile.Chunk != nil {
|
if profile.Chunk != nil {
|
||||||
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), resolution, state)
|
normalizeDocumentBindingReferencePaths(profile.Chunk, path.Key("chunk"), normalizer, state)
|
||||||
}
|
}
|
||||||
if profile.Output != nil {
|
if profile.Output != nil {
|
||||||
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state)
|
normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), normalizer, state)
|
||||||
}
|
}
|
||||||
for laneID, lane := range profile.Artifacts {
|
for laneID, lane := range profile.Artifacts {
|
||||||
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
|
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), normalizer, state)
|
||||||
profile.Artifacts[laneID] = lane
|
profile.Artifacts[laneID] = lane
|
||||||
}
|
}
|
||||||
stepResolution := resolution.inSequence(path.Key("steps"))
|
|
||||||
for index := range profile.Steps {
|
for index := range profile.Steps {
|
||||||
step := &profile.Steps[index]
|
step := &profile.Steps[index]
|
||||||
stepPath := path.Key("steps").Index(index)
|
stepPath := path.Key("steps").Index(index)
|
||||||
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
|
stepNormalizer := normalizer
|
||||||
|
if step.ID != strings.TrimSpace(step.ID) {
|
||||||
|
resolution.RecordNormalization(path.Key("steps"), "normalized step identity")
|
||||||
|
stepNormalizer.covered = true
|
||||||
|
}
|
||||||
|
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepNormalizer, state)
|
||||||
for laneID, lane := range step.Artifacts {
|
for laneID, lane := range step.Artifacts {
|
||||||
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
|
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
|
||||||
step.Artifacts[laneID] = lane
|
step.Artifacts[laneID] = lane
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -59,14 +64,14 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
|
func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
|
||||||
if state == nil || state.mainDirectory != "" {
|
if state == nil || state.mainDirectory != "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
info, ok := resolution.InputInfo(configkit.Keys("version"))
|
info, ok := resolution.EntryInfo(configkit.Keys("version"))
|
||||||
if ok && len(info.Origin.Locations) == 1 {
|
if ok && len(info.Origin.Locations) == 1 {
|
||||||
if source, sourceOK := resolution.Source(info.Origin.Locations[0].SourceID); sourceOK && source.Path != "" {
|
if directory := state.sourceDirectory(info.Origin.Locations[0].SourceID, resolution.Source); directory != "" {
|
||||||
state.mainDirectory = filepath.Dir(source.Path)
|
state.mainDirectory = directory
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -95,23 +100,30 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
|
|||||||
for _, name := range sortedReferenceDocumentKeys(references) {
|
for _, name := range sortedReferenceDocumentKeys(references) {
|
||||||
source := references[name]
|
source := references[name]
|
||||||
sourcePath := path.Key(name).Key("path")
|
sourcePath := path.Key(name).Key("path")
|
||||||
|
if source.Path != nil && resolution.IsExplicit(sourcePath) {
|
||||||
|
value := strings.TrimSpace(*source.Path)
|
||||||
|
if value != *source.Path {
|
||||||
|
resolution.RecordNormalization(sourcePath, "trimmed reference path")
|
||||||
|
source.Path = documentValue(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
|
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
|
||||||
base, ok := documentPathBase(resolution, state, sourcePath)
|
base, ok := documentPathBase(resolution.Resolution, state, sourcePath)
|
||||||
if !ok {
|
if !ok {
|
||||||
resolution.Error(sourcePath, "relative path has no unambiguous source base")
|
resolution.Error(sourcePath, "relative path has no unambiguous source base")
|
||||||
} else {
|
} else {
|
||||||
value := filepath.Clean(filepath.Join(base, *source.Path))
|
value := filepath.Clean(filepath.Join(base, *source.Path))
|
||||||
if value != *source.Path {
|
if value != *source.Path {
|
||||||
resolution.Record(sourcePath, configkit.ChangeNormalization, "resolved external reference path")
|
resolution.RecordNormalization(sourcePath, "resolved external reference path")
|
||||||
source.Path = documentValue(value)
|
source.Path = documentValue(value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
references[name] = source
|
|
||||||
}
|
}
|
||||||
|
references[name] = source
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
||||||
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
|
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -125,16 +137,16 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if *target != nil && **target != value {
|
if *target != nil && **target != value {
|
||||||
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
|
configkit.Normalize(resolution, path, target, documentValue(value), reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
|
func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
|
||||||
original := path
|
original := path
|
||||||
if state != nil {
|
if state != nil {
|
||||||
original = state.originalPath(path)
|
original = state.originalPath(path)
|
||||||
}
|
}
|
||||||
info, ok := resolution.InputInfo(original)
|
info, ok := resolution.EntryInfo(original)
|
||||||
if !ok {
|
if !ok {
|
||||||
if state != nil && state.mainDirectory != "" {
|
if state != nil && state.mainDirectory != "" {
|
||||||
return state.mainDirectory, true
|
return state.mainDirectory, true
|
||||||
@@ -154,14 +166,35 @@ func documentPathBase(resolution documentNormalizer, state *documentLoadState, p
|
|||||||
if len(info.Origin.Locations) != 1 {
|
if len(info.Origin.Locations) != 1 {
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
source, ok := resolution.Source(info.Origin.Locations[0].SourceID)
|
directory := state.sourceDirectory(info.Origin.Locations[0].SourceID, resolution.Source)
|
||||||
if !ok || source.Path == "" {
|
if directory == "" {
|
||||||
if state != nil && state.mainDirectory != "" {
|
if state != nil && state.mainDirectory != "" {
|
||||||
return state.mainDirectory, true
|
return state.mainDirectory, true
|
||||||
}
|
}
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
return filepath.Dir(source.Path), true
|
return directory, true
|
||||||
}
|
}
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Source evidence is fixed during resolution. Cache only the directory needed
|
||||||
|
// by paths, avoiding repeated copies of unrelated import-selection evidence.
|
||||||
|
func (state *documentLoadState) sourceDirectory(id string, lookup func(string) (configkit.SourceInfo, bool)) string {
|
||||||
|
if state != nil {
|
||||||
|
if directory, ok := state.sourceDirectories[id]; ok {
|
||||||
|
return directory
|
||||||
|
}
|
||||||
|
}
|
||||||
|
directory := ""
|
||||||
|
if source, ok := lookup(id); ok && source.Path != "" {
|
||||||
|
directory = filepath.Dir(source.Path)
|
||||||
|
}
|
||||||
|
if state != nil {
|
||||||
|
if state.sourceDirectories == nil {
|
||||||
|
state.sourceDirectories = make(map[string]string)
|
||||||
|
}
|
||||||
|
state.sourceDirectories[id] = directory
|
||||||
|
}
|
||||||
|
return directory
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,9 +5,47 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestSourceDirectoryLookupIsMemoizedPerLoad(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
state := newDocumentLoadState(dir)
|
||||||
|
calls := make(map[string]int)
|
||||||
|
lookup := func(id string) (configkit.SourceInfo, bool) {
|
||||||
|
calls[id]++
|
||||||
|
if id == "missing" {
|
||||||
|
return configkit.SourceInfo{}, false
|
||||||
|
}
|
||||||
|
if id == "memory" {
|
||||||
|
return configkit.SourceInfo{}, true
|
||||||
|
}
|
||||||
|
return configkit.SourceInfo{Path: filepath.Join(dir, id, "config.yml")}, true
|
||||||
|
}
|
||||||
|
for range 2 {
|
||||||
|
for _, id := range []string{"first", "second", "missing", "memory"} {
|
||||||
|
want := ""
|
||||||
|
if id == "first" || id == "second" {
|
||||||
|
want = filepath.Join(dir, id)
|
||||||
|
}
|
||||||
|
if got := state.sourceDirectory(id, lookup); got != want {
|
||||||
|
t.Fatalf("directory(%s) = %q, want %q", id, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for id, count := range calls {
|
||||||
|
if count != 1 {
|
||||||
|
t.Fatalf("source %s queried %d times", id, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other := newDocumentLoadState(dir)
|
||||||
|
if got := other.sourceDirectory("first", func(string) (configkit.SourceInfo, bool) {
|
||||||
|
return configkit.SourceInfo{Path: filepath.Join(dir, "changed", "config.yml")}, true
|
||||||
|
}); got != filepath.Join(dir, "changed") {
|
||||||
|
t.Fatalf("another load reused directory %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -27,7 +65,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(main, []byte(`version: 5
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
configkit:
|
config:
|
||||||
imports:
|
imports:
|
||||||
- path: fragments/pipelines.yml
|
- path: fragments/pipelines.yml
|
||||||
output: {directory: ./output}
|
output: {directory: ./output}
|
||||||
@@ -56,6 +94,51 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
fragmentDir := filepath.Join(root, "fragments")
|
||||||
|
if err := os.Mkdir(fragmentDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(root, "notarius.yml")
|
||||||
|
fragment := filepath.Join(fragmentDir, "pipeline.yml")
|
||||||
|
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fragment, []byte(`pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
steps:
|
||||||
|
- id: " first "
|
||||||
|
references:
|
||||||
|
" source ": {path: " ./source.txt "}
|
||||||
|
artifacts:
|
||||||
|
result:
|
||||||
|
extract: {module: " extract "}
|
||||||
|
- id: second
|
||||||
|
artifacts:
|
||||||
|
next:
|
||||||
|
extract: {module: " extract "}
|
||||||
|
`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") {
|
||||||
|
t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
|
||||||
|
}
|
||||||
|
step := loaded.Config.Pipelines["session"].Steps[0]
|
||||||
|
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
|
||||||
|
t.Fatalf("normalized step = %#v", step)
|
||||||
|
}
|
||||||
|
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(1).Key("artifacts").Key("next").Key("extract").Key("module"))
|
||||||
|
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Fatalf("module leaf history = %#v", history)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -77,6 +160,37 @@ func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiscoveredSymlinkUsesCanonicalMainDirectory(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
realDir := filepath.Join(root, "real")
|
||||||
|
if err := os.Mkdir(realDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(realDir, "main.yml")
|
||||||
|
if err := os.WriteFile(main, []byte("version: 5\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
link := filepath.Join(root, "notarius.yml")
|
||||||
|
if err := os.Symlink(main, link); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(link), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(realDir, "notarius-output") {
|
||||||
|
t.Fatalf("canonical source = %q, output = %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequiredSearchExhaustion(t *testing.T) {
|
||||||
|
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||||
|
_, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(missing), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("missing required search succeeded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDocumentPathsUseOverlayDeclarationDirectory(t *testing.T) {
|
func TestDocumentPathsUseOverlayDeclarationDirectory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDocumentLoadAcceptsVersion5AndCanonicalShapes(t *testing.T) {
|
func TestDocumentLoadAcceptsVersion5AndCanonicalShapes(t *testing.T) {
|
||||||
|
|||||||
622
internal/core/config/inheritance_test.go
Normal file
622
internal/core/config/inheritance_test.go
Normal file
@@ -0,0 +1,622 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPipelineInheritancePreservesAndUpsertsSteps(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
childSteps string
|
||||||
|
wantStepIDs []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "inherits ordered steps",
|
||||||
|
wantStepIDs: []string{"prepare", "summarize"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "appends distinct ordered steps",
|
||||||
|
childSteps: `
|
||||||
|
steps:
|
||||||
|
- id: replacement
|
||||||
|
artifacts:
|
||||||
|
summary:
|
||||||
|
extract: {module: extract}`,
|
||||||
|
wantStepIDs: []string{"prepare", "summarize", "replacement"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
source := `version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
steps:
|
||||||
|
- id: prepare
|
||||||
|
artifacts:
|
||||||
|
score:
|
||||||
|
extract: {module: extract, retries: 2}
|
||||||
|
- id: summarize
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
extract: {module: extract}
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
llm_profile: local
|
||||||
|
structured_output_repair_attempts: 0` + test.childSteps + "\n"
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("inheritance.yml", []byte(source)), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadConfigSource() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := loaded.Config.Pipelines["parent"]
|
||||||
|
child := loaded.Config.Pipelines["child"]
|
||||||
|
if parent.Input.Module != "input" || child.Input.Module != "input" {
|
||||||
|
t.Fatalf("parent/child inputs = %#v / %#v", parent.Input, child.Input)
|
||||||
|
}
|
||||||
|
if child.LLMProfile != "local" ||
|
||||||
|
child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("child overrides = %#v", child)
|
||||||
|
}
|
||||||
|
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 2 {
|
||||||
|
t.Fatalf("parent settings changed: %#v", parent)
|
||||||
|
}
|
||||||
|
if len(child.Steps) != len(test.wantStepIDs) {
|
||||||
|
t.Fatalf("child steps = %#v, want %v", child.Steps, test.wantStepIDs)
|
||||||
|
}
|
||||||
|
for i, want := range test.wantStepIDs {
|
||||||
|
if child.Steps[i].ID != want {
|
||||||
|
t.Fatalf("child step %d = %q, want %q", i, child.Steps[i].ID, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if test.childSteps == "" && child.Steps[0].Artifacts["score"].Extract.Module != "extract" {
|
||||||
|
t.Fatalf("inherited first artifact = %#v", child.Steps[0].Artifacts["score"])
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, id := range []string{"parent", "child"} {
|
||||||
|
resolved, err := loaded.Config.Resolve(ResolveInput{PipelineID: id, Catalog: effectiveCatalog(t)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve(%q) error = %v", id, err)
|
||||||
|
}
|
||||||
|
if resolved.ResolvedPipeline.ID != id {
|
||||||
|
t.Fatalf("resolved pipeline ID = %q, want %q", resolved.ResolvedPipeline.ID, id)
|
||||||
|
}
|
||||||
|
wantResolvedSteps := len(test.wantStepIDs)
|
||||||
|
if id == "parent" {
|
||||||
|
wantResolvedSteps = 2
|
||||||
|
}
|
||||||
|
if len(resolved.ResolvedPipeline.Steps) != wantResolvedSteps {
|
||||||
|
t.Fatalf("resolved %q steps = %#v", id, resolved.ResolvedPipeline.Steps)
|
||||||
|
}
|
||||||
|
if id == "child" && resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].Extract.LLMProfile != "local" {
|
||||||
|
t.Fatalf("resolved child profile = %q", resolved.ResolvedPipeline.Steps[0].ArtifactLanes[0].Extract.LLMProfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArtifactLaneInheritanceUsesSiblingMappingComposition(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("lanes.yml", []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
input: {module: input}
|
||||||
|
artifacts:
|
||||||
|
base:
|
||||||
|
extract: {module: extract, retries: 2}
|
||||||
|
derived:
|
||||||
|
extends: base
|
||||||
|
extract: {retries: 4}
|
||||||
|
`)), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadConfigSource() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lanes := loaded.Config.Pipelines["session"].Artifacts
|
||||||
|
if lanes["base"].Extract.Module != "extract" || lanes["base"].Extract.Retries != 2 {
|
||||||
|
t.Fatalf("base lane = %#v", lanes["base"])
|
||||||
|
}
|
||||||
|
if lanes["derived"].Extract.Module != "extract" || lanes["derived"].Extract.Retries != 4 {
|
||||||
|
t.Fatalf("derived lane = %#v", lanes["derived"])
|
||||||
|
}
|
||||||
|
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}); err != nil {
|
||||||
|
t.Fatalf("Resolve() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInheritedPresenceOverridesRemainExplicit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("presence.yml", []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
structured_output_repair_attempts: 3
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
validators: []
|
||||||
|
artifacts:
|
||||||
|
score:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
validators: [{module: configured-a}]
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
artifacts:
|
||||||
|
score:
|
||||||
|
extract:
|
||||||
|
validators: []
|
||||||
|
`)), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
var loadError *configkit.LoadError
|
||||||
|
if errors.As(err, &loadError) {
|
||||||
|
t.Fatalf("loadConfigSource() diagnostics = %#v", loadError.Diagnostics())
|
||||||
|
}
|
||||||
|
t.Fatalf("loadConfigSource() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := loaded.Config.Pipelines["parent"]
|
||||||
|
child := loaded.Config.Pipelines["child"]
|
||||||
|
if parent.Chunk.Module != "generic" || !parent.Chunk.Validators.Set || parent.Chunk.Validators.Validators == nil || len(parent.Chunk.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("parent empty chunk validators = %#v", parent.Chunk)
|
||||||
|
}
|
||||||
|
if !parent.Artifacts["score"].Extract.Validators.Set || len(parent.Artifacts["score"].Extract.Validators.Validators) != 1 ||
|
||||||
|
parent.Artifacts["score"].Extract.Validators.Validators[0].Module != "configured-a" {
|
||||||
|
t.Fatalf("parent extraction validators = %#v", parent.Artifacts["score"].Extract.Validators)
|
||||||
|
}
|
||||||
|
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 3 {
|
||||||
|
t.Fatalf("parent repair budget = %#v", parent.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("child zero repair budget = %#v", child.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
extract := child.Artifacts["score"].Extract
|
||||||
|
if !extract.Validators.Set || extract.Validators.Validators == nil || len(extract.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("child empty extraction validators = %#v", extract.Validators)
|
||||||
|
}
|
||||||
|
if !child.Chunk.Validators.Set || child.Chunk.Validators.Validators == nil || len(child.Chunk.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("inherited empty chunk validators = %#v", child.Chunk.Validators)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve(child) error = %v", err)
|
||||||
|
}
|
||||||
|
lanes := resolved.ResolvedPipeline.AllArtifactLanes()
|
||||||
|
if len(lanes) != 1 || lanes[0].Extract.StructuredOutputRepairAttempts == nil || *lanes[0].Extract.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("resolved child repair budget = %#v", lanes)
|
||||||
|
}
|
||||||
|
if !resolved.ResolvedPipeline.Chunk.Validators.Set || len(resolved.ResolvedPipeline.Chunk.Validators.Validators) != 0 ||
|
||||||
|
!lanes[0].Extract.Validators.Set || len(lanes[0].Extract.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("resolved empty validator overrides = chunk %#v, extraction %#v", resolved.ResolvedPipeline.Chunk.Validators, lanes[0].Extract.Validators)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInheritedFalseOptionRetainsExplicitMetadata(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("false-option.yml", []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
output:
|
||||||
|
module: json
|
||||||
|
options:
|
||||||
|
evidence_context: {enabled: false}
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
`)), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadConfigSource() error = %v", err)
|
||||||
|
}
|
||||||
|
options := loaded.Config.Pipelines["child"].Output.Options["evidence_context"].(map[string]any)
|
||||||
|
if enabled, ok := options["enabled"].(bool); !ok || enabled {
|
||||||
|
t.Fatalf("inherited evidence_context.enabled = %#v", options["enabled"])
|
||||||
|
}
|
||||||
|
path := configkit.Keys("pipelines", "child", "output", "options", "evidence_context", "enabled")
|
||||||
|
info, ok := loaded.result.Metadata().Lookup(path)
|
||||||
|
if !ok || !info.Explicit || info.Null || len(info.Origin.Inheritance) == 0 {
|
||||||
|
t.Fatalf("inherited false metadata = %#v, found=%v", info, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInheritedAndDirectNullRepairBudgetsRemainInvalid(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
path string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "child null",
|
||||||
|
source: `version: 5
|
||||||
|
pipelines:
|
||||||
|
parent: {input: {module: input}, structured_output_repair_attempts: 2}
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
structured_output_repair_attempts: null
|
||||||
|
`,
|
||||||
|
path: "/pipelines/child/structured_output_repair_attempts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "inherited null",
|
||||||
|
source: `version: 5
|
||||||
|
pipelines:
|
||||||
|
parent: {input: {module: input}, structured_output_repair_attempts: null}
|
||||||
|
child: {extends: parent}
|
||||||
|
`,
|
||||||
|
path: "/pipelines/parent/structured_output_repair_attempts",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
_, err := loadConfigSource(configkit.Bytes("null.yml", []byte(test.source)), LoadOptions{})
|
||||||
|
assertDocumentDiagnosticAt(t, err, test.path)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverlayInheritancePrecedesOverrides(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
overlays := filepath.Join(root, "overlays")
|
||||||
|
if err := os.MkdirAll(overlays, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(root, "config.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
structured_output_repair_attempts: 1
|
||||||
|
inherited:
|
||||||
|
extends: parent
|
||||||
|
overridden:
|
||||||
|
extends: parent
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
cancelled:
|
||||||
|
extends: parent
|
||||||
|
input: {module: input}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte(`pipelines:
|
||||||
|
parent:
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
cancelled:
|
||||||
|
extends: null
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := Load(main, LoadOptions{Overlay: "development"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := *loaded.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("overlay-inherited repair budget = %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *loaded.Config.Pipelines["overridden"].StructuredOutputRepairAttempts; got != 0 {
|
||||||
|
t.Fatalf("child replacement repair budget = %d, want 0", got)
|
||||||
|
}
|
||||||
|
if loaded.Config.Pipelines["cancelled"].StructuredOutputRepairAttempts != nil {
|
||||||
|
t.Fatalf("cancelled child inherited parent settings: %#v", loaded.Config.Pipelines["cancelled"])
|
||||||
|
}
|
||||||
|
|
||||||
|
withOverride, err := Load(main, LoadOptions{
|
||||||
|
Overlay: "development",
|
||||||
|
Overrides: []configkit.Override{{
|
||||||
|
Path: configkit.Keys("pipelines", "parent", "structured_output_repair_attempts"),
|
||||||
|
Value: 3,
|
||||||
|
Flag: "--repair-budget",
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() with explicit override error = %v", err)
|
||||||
|
}
|
||||||
|
if got := *withOverride.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
|
||||||
|
t.Fatalf("inherited repair budget after parent override = %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := *withOverride.Config.Pipelines["parent"].StructuredOutputRepairAttempts; got != 3 {
|
||||||
|
t.Fatalf("parent override repair budget = %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
parentDirectory := filepath.Join(root, "parents")
|
||||||
|
childDirectory := filepath.Join(root, "children")
|
||||||
|
overlayDirectory := filepath.Join(root, "overlays")
|
||||||
|
fixtureDirectory := filepath.Join(root, "fixtures")
|
||||||
|
for _, directory := range []string{parentDirectory, childDirectory, overlayDirectory, fixtureDirectory} {
|
||||||
|
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, fixture := range []struct{ directory, name string }{
|
||||||
|
{parentDirectory, "shared.txt"},
|
||||||
|
{parentDirectory, "parent.txt"},
|
||||||
|
{parentDirectory, "step.txt"},
|
||||||
|
{childDirectory, "child.txt"},
|
||||||
|
{overlayDirectory, "overlay.txt"},
|
||||||
|
{fixtureDirectory, "outside.txt"},
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(fixture.directory, fixture.name), []byte("fixture\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := filepath.Join(parentDirectory, "pipelines.yml")
|
||||||
|
if err := os.WriteFile(parent, []byte(`pipelines:
|
||||||
|
" parent ":
|
||||||
|
input: {module: input}
|
||||||
|
references:
|
||||||
|
" shared ": {path: ./shared.txt}
|
||||||
|
nested:
|
||||||
|
extends: " shared "
|
||||||
|
source: {path: ./parent.txt}
|
||||||
|
outside: {path: ../fixtures/outside.txt}
|
||||||
|
steps:
|
||||||
|
- id: first
|
||||||
|
references:
|
||||||
|
step-source: {path: ./step.txt}
|
||||||
|
artifacts:
|
||||||
|
summary:
|
||||||
|
extract: {module: extract}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
child := filepath.Join(childDirectory, "pipelines.yml")
|
||||||
|
if err := os.WriteFile(child, []byte(`pipelines:
|
||||||
|
child:
|
||||||
|
extends: " parent "
|
||||||
|
references:
|
||||||
|
source: {path: ./child.txt}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(root, "notarius.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: parents/pipelines.yml
|
||||||
|
- path: children/pipelines.yml
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(overlayDirectory, "development.yml"), []byte(`pipelines:
|
||||||
|
" parent ":
|
||||||
|
references:
|
||||||
|
" shared ": {path: ./overlay.txt}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := loadConfig(main, LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
assertReferencePath := func(pipelineID, referenceID, want string) {
|
||||||
|
t.Helper()
|
||||||
|
if got := loaded.Config.Pipelines[pipelineID].References[referenceID].Path; got != want {
|
||||||
|
t.Errorf("%s reference %s path = %q, want %q", pipelineID, referenceID, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertReferencePath("parent", "shared", filepath.Join(parentDirectory, "shared.txt"))
|
||||||
|
assertReferencePath("child", "shared", filepath.Join(parentDirectory, "shared.txt"))
|
||||||
|
assertReferencePath("child", "nested", filepath.Join(parentDirectory, "shared.txt"))
|
||||||
|
assertReferencePath("parent", "source", filepath.Join(parentDirectory, "parent.txt"))
|
||||||
|
assertReferencePath("child", "source", filepath.Join(childDirectory, "child.txt"))
|
||||||
|
// Keep ../ coverage separate from paths that distinguish the declaring file.
|
||||||
|
assertReferencePath("child", "outside", filepath.Join(fixtureDirectory, "outside.txt"))
|
||||||
|
if got := loaded.Config.Pipelines["child"].Steps[0].References["step-source"].Path; got != filepath.Join(parentDirectory, "step.txt") {
|
||||||
|
t.Errorf("inherited step reference path = %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
overlaid, err := loadConfig(main, LoadOptions{Overlay: "development"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadConfig() with overlay error = %v", err)
|
||||||
|
}
|
||||||
|
for _, pipelineID := range []string{"parent", "child"} {
|
||||||
|
for _, referenceID := range []string{"shared", "nested"} {
|
||||||
|
if got, want := overlaid.Config.Pipelines[pipelineID].References[referenceID].Path, filepath.Join(overlayDirectory, "overlay.txt"); got != want {
|
||||||
|
t.Errorf("overlaid %s reference %s path = %q, want %q", pipelineID, referenceID, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got, want := overlaid.Config.Pipelines["child"].References["source"].Path, filepath.Join(childDirectory, "child.txt"); got != want {
|
||||||
|
t.Errorf("child replacement path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := overlaid.Config.Pipelines["child"].Steps[0].References["step-source"].Path, filepath.Join(parentDirectory, "step.txt"); got != want {
|
||||||
|
t.Errorf("overlaid inherited step reference path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
parentDirectory := filepath.Join(root, "parents")
|
||||||
|
childDirectory := filepath.Join(root, "children")
|
||||||
|
overlayDirectory := filepath.Join(root, "overlays")
|
||||||
|
for _, directory := range []string{parentDirectory, childDirectory, overlayDirectory} {
|
||||||
|
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parentFile := filepath.Join(parentDirectory, "pipelines.yml")
|
||||||
|
if err := os.WriteFile(parentFile, []byte(`pipelines:
|
||||||
|
parent:
|
||||||
|
input: {module: input}
|
||||||
|
structured_output_repair_attempts: 2
|
||||||
|
references:
|
||||||
|
source: {path: ./shared.txt}
|
||||||
|
artifacts:
|
||||||
|
summary:
|
||||||
|
extract:
|
||||||
|
module: extract
|
||||||
|
options:
|
||||||
|
nested: {value: original}
|
||||||
|
credentials: {api_key: fake-parent-secret}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(parentDirectory, "shared.txt"), []byte("source\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
childFile := filepath.Join(childDirectory, "pipelines.yml")
|
||||||
|
if err := os.WriteFile(childFile, []byte(`pipelines:
|
||||||
|
child:
|
||||||
|
extends: parent
|
||||||
|
structured_output_repair_attempts: 0
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(root, "notarius.yml")
|
||||||
|
if err := os.WriteFile(main, []byte(`version: 5
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: parents/pipelines.yml
|
||||||
|
- path: children/pipelines.yml
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(overlayDirectory, "development.yml"), []byte(`pipelines:
|
||||||
|
parent:
|
||||||
|
artifacts:
|
||||||
|
summary:
|
||||||
|
extract:
|
||||||
|
options:
|
||||||
|
credentials: {api_key: fake-overlay-secret}
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := Load(main, LoadOptions{Overlay: "development"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
child := loaded.Config.Pipelines["child"]
|
||||||
|
wantPath := filepath.Join(parentDirectory, "shared.txt")
|
||||||
|
if got := child.References["source"].Path; got != wantPath {
|
||||||
|
t.Fatalf("runtime inherited path = %q, want %q", got, wantPath)
|
||||||
|
}
|
||||||
|
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
|
||||||
|
t.Fatalf("runtime inherited repair setting = %#v", child.StructuredOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := loaded.WriteEffective(&effective); err != nil {
|
||||||
|
t.Fatalf("WriteEffective() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(effective.String(), wantPath) || strings.Contains(effective.String(), "extends:") ||
|
||||||
|
!strings.Contains(effective.String(), "structured_output_repair_attempts: 0") {
|
||||||
|
t.Fatalf("effective configuration is inconsistent with runtime conversion: %s", effective.String())
|
||||||
|
}
|
||||||
|
for _, secret := range []string{"fake-parent-secret", "fake-overlay-secret"} {
|
||||||
|
if strings.Contains(effective.String(), secret) {
|
||||||
|
t.Fatalf("effective configuration exposed %q", secret)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
path configkit.Path
|
||||||
|
}{
|
||||||
|
{name: "repair setting", path: configkit.Keys("pipelines", "child", "structured_output_repair_attempts")},
|
||||||
|
{name: "normalized reference", path: configkit.Keys("pipelines", "child", "references", "source", "path")},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
explanation, ok := loaded.result.Explain(test.path)
|
||||||
|
if !ok || len(explanation.History) == 0 {
|
||||||
|
t.Fatalf("inherited explanation = %#v, found=%v", explanation, ok)
|
||||||
|
}
|
||||||
|
inheritanceFromParent := false
|
||||||
|
for _, change := range explanation.History {
|
||||||
|
if len(change.Origin.Inheritance) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, location := range change.Origin.Locations {
|
||||||
|
for _, source := range loaded.result.Sources() {
|
||||||
|
if source.ID == location.SourceID && source.Path == parentFile {
|
||||||
|
inheritanceFromParent = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !inheritanceFromParent {
|
||||||
|
t.Fatalf("inheritance history lacks parent source evidence: %#v", explanation.History)
|
||||||
|
}
|
||||||
|
var rendered bytes.Buffer
|
||||||
|
if err := loaded.WriteExplanation(&rendered, test.path.String()); err != nil {
|
||||||
|
t.Fatalf("WriteExplanation() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rendered.String(), filepath.ToSlash(filepath.Join("parents", "pipelines.yml"))) {
|
||||||
|
t.Fatalf("explanation lacks parent source evidence: %s", rendered.String())
|
||||||
|
}
|
||||||
|
if test.name == "normalized reference" {
|
||||||
|
foundNormalization := false
|
||||||
|
for _, change := range explanation.History {
|
||||||
|
if change.Origin.Operation == configkit.ChangeNormalization {
|
||||||
|
foundNormalization = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundNormalization || !strings.Contains(rendered.String(), wantPath) {
|
||||||
|
t.Fatalf("normalized path evidence = %#v; rendered %s", explanation.History, rendered.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
secretPath := configkit.Keys("pipelines", "child", "artifacts", "summary", "extract", "options", "credentials", "api_key")
|
||||||
|
secretExplanation, ok := loaded.result.Explain(secretPath)
|
||||||
|
if !ok || !secretExplanation.Effective.Redacted {
|
||||||
|
t.Fatalf("inherited secret explanation = %#v, found=%v", secretExplanation, ok)
|
||||||
|
}
|
||||||
|
redactedHistory := false
|
||||||
|
for _, change := range secretExplanation.History {
|
||||||
|
redactedHistory = redactedHistory || change.Before.Redacted || change.After.Redacted
|
||||||
|
}
|
||||||
|
if !redactedHistory {
|
||||||
|
t.Fatalf("secret history was not redacted: %#v", secretExplanation.History)
|
||||||
|
}
|
||||||
|
var secretOutput bytes.Buffer
|
||||||
|
if err := loaded.WriteExplanation(&secretOutput, secretPath.String()); err != nil {
|
||||||
|
t.Fatalf("WriteExplanation(secret) error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(secretOutput.String(), "fake-parent-secret") || strings.Contains(secretOutput.String(), "fake-overlay-secret") ||
|
||||||
|
!strings.Contains(secretOutput.String(), "[REDACTED]") {
|
||||||
|
t.Fatalf("secret explanation did not redact values and history: %s", secretOutput.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
childOptions := loaded.Config.Pipelines["child"].Artifacts["summary"].Extract.Options
|
||||||
|
childOptions["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
parentValue := loaded.Config.Pipelines["parent"].Artifacts["summary"].Extract.Options["nested"].(map[string]any)["value"]
|
||||||
|
if parentValue != "original" {
|
||||||
|
t.Fatalf("child runtime mutation changed parent value: %#v", parentValue)
|
||||||
|
}
|
||||||
|
var frozen bytes.Buffer
|
||||||
|
if err := loaded.WriteEffective(&frozen); err != nil {
|
||||||
|
t.Fatalf("WriteEffective() after runtime mutation error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(frozen.String(), "original") || strings.Contains(frozen.String(), "changed") {
|
||||||
|
t.Fatalf("runtime mutation changed frozen inspection: %s", frozen.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LoadOptions contains configuration selections supplied by an application
|
// LoadOptions contains configuration selections supplied by an application
|
||||||
@@ -57,7 +57,14 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document]
|
|||||||
Path: configkit.Keys("version"),
|
Path: configkit.Keys("version"),
|
||||||
Check: validateDocumentVersion,
|
Check: validateDocumentVersion,
|
||||||
},
|
},
|
||||||
Secret: sensitiveDocumentPath,
|
Sensitive: sensitiveDocumentPath,
|
||||||
|
SequenceIdentities: []configkit.SequenceIdentity{
|
||||||
|
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
|
||||||
|
configkit.SequenceLiteralKey("pipelines"),
|
||||||
|
configkit.SequenceAnyMappingKey(),
|
||||||
|
configkit.SequenceLiteralKey("steps"),
|
||||||
|
}, "id"),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LoadedConfig keeps the finalized runtime configuration together with the
|
// LoadedConfig keeps the finalized runtime configuration together with the
|
||||||
@@ -59,6 +59,28 @@ func Load(path string, options LoadOptions) (*LoadedConfig, error) {
|
|||||||
return loadConfig(path, options)
|
return loadConfig(path, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadSource loads a selected source, including its imports, into an owned configuration.
|
||||||
|
func LoadSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
||||||
|
result, err := loadDocumentWithSource(source, options, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return newLoadedConfig(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MainSourcePath returns the selected main file's recorded path, if any.
|
||||||
|
func (loaded *LoadedConfig) MainSourcePath() string {
|
||||||
|
if loaded == nil || loaded.result == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, source := range loaded.result.Sources() {
|
||||||
|
if source.Layer == configkit.LayerMain && source.Kind == configkit.SourceKindFile && source.ImportedFrom == nil {
|
||||||
|
return source.Path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
||||||
result, err := loadDocumentSource(source, options)
|
result, err := loadDocumentSource(source, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -2,16 +2,47 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
units "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
units "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestLoadSourceRetainsMainFileAndLoadsImports(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
main := filepath.Join(dir, "main.yml")
|
||||||
|
fragment := filepath.Join(dir, "fragment.yml")
|
||||||
|
if err := os.WriteFile(fragment, []byte("output:\n directory: imported-output\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadSource(configkit.File(main), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := loaded.MainSourcePath(); got != main {
|
||||||
|
t.Fatalf("main source path = %q, want %q", got, main)
|
||||||
|
}
|
||||||
|
if got := loaded.Config.Output.Directory; got != filepath.Join(dir, "imported-output") {
|
||||||
|
t.Fatalf("imported output directory = %q", got)
|
||||||
|
}
|
||||||
|
bytesLoaded, err := loadConfigSource(configkit.Bytes("memory.yml", []byte("version: 5\n")), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := bytesLoaded.MainSourcePath(); got != "" {
|
||||||
|
t.Fatalf("non-file main source path = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
|
func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -166,6 +197,33 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNestedConfigOptionRemainsApplicationData(t *testing.T) {
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("options.yml", []byte(`version: 5
|
||||||
|
pipelines:
|
||||||
|
session:
|
||||||
|
input:
|
||||||
|
module: input
|
||||||
|
options:
|
||||||
|
config: {imports: ordinary-data}
|
||||||
|
`)), LoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value := loaded.Config.Pipelines["session"].Input.Options["config"].(map[string]any)["imports"]
|
||||||
|
if value != "ordinary-data" {
|
||||||
|
t.Fatalf("nested config option = %#v", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverrideCannotRepairTypedDecodeFailure(t *testing.T) {
|
||||||
|
_, err := loadConfigSource(configkit.Bytes("invalid.yml", []byte("version: 5\nconcurrency: {total_llm: invalid}\n")), LoadOptions{
|
||||||
|
Overrides: []configkit.Override{{Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm"}},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("override unexpectedly repaired invalid typed input")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedConfigRedactsSensitiveNestedOptions(t *testing.T) {
|
func TestLoadedConfigRedactsSensitiveNestedOptions(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func resolveDocumentOperations(document *Document, resolution *configkit.Resolution, state *documentLoadState) error {
|
func resolveDocumentOperations(document *Document, resolution *configkit.Resolution, state *documentLoadState) error {
|
||||||
@@ -16,8 +16,8 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
|
|||||||
}
|
}
|
||||||
normalizeDocumentPromptKit(document.PromptKit, resolution)
|
normalizeDocumentPromptKit(document.PromptKit, resolution)
|
||||||
normalizeDocumentState(document, resolution)
|
normalizeDocumentState(document, resolution)
|
||||||
normalizeDocumentPipelines(document, resolution, state)
|
|
||||||
normalizeDocumentPaths(document, resolution, state)
|
normalizeDocumentPaths(document, resolution, state)
|
||||||
|
normalizeDocumentPipelines(document, resolution, state)
|
||||||
if document.Concurrency != nil {
|
if document.Concurrency != nil {
|
||||||
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
|
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
|
||||||
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
|
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestWorkerKeyNormalizationPreservesExplicitCount(t *testing.T) {
|
func TestWorkerKeyNormalizationPreservesExplicitCount(t *testing.T) {
|
||||||
|
|||||||
@@ -4,15 +4,14 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// documentLoadState keeps the spelling supplied by the user while resolver
|
// documentLoadState keeps source-directory lookups and original map-key
|
||||||
// operations move map entries to their canonical keys. It is intentionally
|
// spellings local to one load. It never becomes runtime configuration state.
|
||||||
// load-local: source-location handling consumes it during a later resolver
|
|
||||||
// pass and it never becomes runtime configuration state.
|
|
||||||
type documentLoadState struct {
|
type documentLoadState struct {
|
||||||
|
sourceDirectories map[string]string
|
||||||
originalPaths map[string]configkit.Path
|
originalPaths map[string]configkit.Path
|
||||||
mainDirectory string
|
mainDirectory string
|
||||||
invocationDirectory string
|
invocationDirectory string
|
||||||
@@ -65,7 +64,6 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
|
|||||||
}
|
}
|
||||||
validateDocumentPipelinePresence(document, resolution)
|
validateDocumentPipelinePresence(document, resolution)
|
||||||
normalizer := documentNormalizer{Resolution: resolution}
|
normalizer := documentNormalizer{Resolution: resolution}
|
||||||
|
|
||||||
pipelinesPath := configkit.Keys("pipelines")
|
pipelinesPath := configkit.Keys("pipelines")
|
||||||
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
||||||
profile := document.Pipelines[id]
|
profile := document.Pipelines[id]
|
||||||
@@ -192,18 +190,19 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
|
|||||||
profile.Artifacts[laneID] = lane
|
profile.Artifacts[laneID] = lane
|
||||||
}
|
}
|
||||||
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
|
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
|
||||||
resolution = resolution.inSequence(path.Key("steps"))
|
|
||||||
for index := range profile.Steps {
|
for index := range profile.Steps {
|
||||||
step := &profile.Steps[index]
|
step := &profile.Steps[index]
|
||||||
stepPath := path.Key("steps").Index(index)
|
stepPath := path.Key("steps").Index(index)
|
||||||
normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
|
stepNormalizer := resolution
|
||||||
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state)
|
stepNormalizer.covered = step.ID != strings.TrimSpace(step.ID)
|
||||||
|
normalizeDocumentValue(stepNormalizer, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
|
||||||
|
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), stepNormalizer, state)
|
||||||
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
|
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
|
||||||
lane := step.Artifacts[laneID]
|
lane := step.Artifacts[laneID]
|
||||||
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
|
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), stepNormalizer, state)
|
||||||
step.Artifacts[laneID] = lane
|
step.Artifacts[laneID] = lane
|
||||||
}
|
}
|
||||||
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state)
|
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", stepNormalizer, state)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,7 +214,6 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
|
|||||||
if lane.Normalize != nil {
|
if lane.Normalize != nil {
|
||||||
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
|
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
|
||||||
}
|
}
|
||||||
resolution = resolution.inSequence(path.Key("validators"))
|
|
||||||
for index := range lane.Validators {
|
for index := range lane.Validators {
|
||||||
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
|
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
|
||||||
}
|
}
|
||||||
@@ -225,7 +223,6 @@ func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, res
|
|||||||
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
|
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
|
||||||
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
|
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
|
||||||
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
|
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
|
||||||
resolution = resolution.inSequence(path.Key("validators"))
|
|
||||||
for index := range binding.Validators {
|
for index := range binding.Validators {
|
||||||
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
|
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
|
||||||
}
|
}
|
||||||
@@ -255,7 +252,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if *target != value {
|
if *target != value {
|
||||||
resolution.Record(path, configkit.ChangeNormalization, reason)
|
resolution.RecordNormalization(path, reason)
|
||||||
*target = value
|
*target = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -266,7 +263,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk
|
|||||||
}
|
}
|
||||||
value := strings.TrimSpace(**target)
|
value := strings.TrimSpace(**target)
|
||||||
if **target != value {
|
if **target != value {
|
||||||
resolution.Record(path, configkit.ChangeNormalization, reason)
|
resolution.RecordNormalization(path, reason)
|
||||||
**target = value
|
**target = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -312,7 +309,7 @@ func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if changed {
|
if changed {
|
||||||
resolution.Record(path, configkit.ChangeNormalization, "normalized "+kind+" keys", dependencies...)
|
resolution.RecordNormalization(path, "normalized "+kind+" keys", dependencies...)
|
||||||
values = result
|
values = result
|
||||||
}
|
}
|
||||||
return values
|
return values
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDocumentRejectsExplicitInvalidBranches(t *testing.T) {
|
func TestDocumentRejectsExplicitInvalidBranches(t *testing.T) {
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline {
|
func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline {
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ func TestSequenceNormalizationPreservesRuntimeAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
if !strings.Contains(effective.String(), test.want) ||
|
if !strings.Contains(effective.String(), test.want) ||
|
||||||
!strings.Contains(explanation.String(), test.want) ||
|
!strings.Contains(explanation.String(), test.want) ||
|
||||||
!strings.Contains(explanation.String(), path) ||
|
(test.name != "step identifier" && !strings.Contains(explanation.String(), path)) ||
|
||||||
!strings.Contains(explanation.String(), "normalization") {
|
!strings.Contains(explanation.String(), "normalization") {
|
||||||
t.Fatalf("missing effective value or provenance:\n%s\n%s", effective.String(), explanation.String())
|
t.Fatalf("missing effective value or provenance:\n%s\n%s", effective.String(), explanation.String())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
|
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/readonlyfs"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/readonlyfs"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
@@ -57,8 +58,8 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
|
// RegisterFallbackProfileFS registers lower-precedence application profiles.
|
||||||
// when an operator-configured source does not provide a matching profile.
|
// Operator entries recursively overlay them, retaining omitted fields.
|
||||||
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
|
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("asset registry must not be nil")
|
return fmt.Errorf("asset registry must not be nil")
|
||||||
@@ -99,43 +100,45 @@ func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
|
|||||||
return digest, err
|
return digest, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
|
// PromptKitConfigurationSources returns the registered prompt and fallback
|
||||||
options, _, err := r.promptKitOptions()
|
// profile assets in one strict native PromptKit underlay. Schema assets remain a
|
||||||
return options, err
|
// separate PromptKit option because they are runtime validation resources.
|
||||||
}
|
func (r *AssetRegistry) PromptKitConfigurationSources() (promptkit.ConfigurationSources, []promptkit.Option, error) {
|
||||||
|
if r == nil {
|
||||||
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
|
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("asset registry must not be nil")
|
||||||
promptFS, err := r.PromptFS()
|
}
|
||||||
|
inputs, err := promptKitFilesystemInputs(r.prompts, "notarius-prompts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
|
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare prompt configuration sources: %w", err)
|
||||||
|
}
|
||||||
|
profiles, err := promptKitFilesystemInputs(r.fallbackProfiles, "notarius-fallback-profiles")
|
||||||
|
if err != nil {
|
||||||
|
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare fallback profile configuration sources: %w", err)
|
||||||
|
}
|
||||||
|
inputs = append(inputs, profiles...)
|
||||||
|
var underlays []configkit.Underlay
|
||||||
|
if len(inputs) != 0 {
|
||||||
|
underlays = []configkit.Underlay{{Inputs: inputs}}
|
||||||
}
|
}
|
||||||
schemaFS, err := r.SchemaFS()
|
schemaFS, err := r.SchemaFS()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
|
return promptkit.ConfigurationSources{}, nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||||
}
|
}
|
||||||
options := []promptkit.Option{
|
return promptkit.ConfigurationSources{Underlays: underlays}, []promptkit.Option{
|
||||||
promptkit.WithPromptFS(promptFS, "."),
|
|
||||||
promptkit.WithSchemaFS(schemaFS, "."),
|
promptkit.WithSchemaFS(schemaFS, "."),
|
||||||
}
|
}, nil
|
||||||
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
if hasFallback {
|
|
||||||
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
|
|
||||||
}
|
|
||||||
return options, fallbackDigest, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
|
func promptKitFilesystemInputs(sources []AssetSource, label string) ([]configkit.Input, error) {
|
||||||
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
|
var selected []configkit.Input
|
||||||
if err != nil {
|
for index, source := range sources {
|
||||||
return nil, false, err
|
inputs, err := promptkit.FilesystemInputs(fmt.Sprintf("%s-%d", label, index), source.FS, source.Root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("select filesystem inputs: %w", err)
|
||||||
|
}
|
||||||
|
selected = append(selected, inputs...)
|
||||||
}
|
}
|
||||||
if !hasFallback {
|
return selected, nil
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
|
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
|
||||||
|
|||||||
53
internal/framework/llm/asset_registry_sources_test.go
Normal file
53
internal/framework/llm/asset_registry_sources_test.go
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplicationAssetsComposeStrictlyBeforeOperatorOverlay(t *testing.T) {
|
||||||
|
for _, conflict := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint("conflict=", conflict), func(t *testing.T) {
|
||||||
|
registry := NewAssetRegistry()
|
||||||
|
for _, id := range []string{"first", "second"} {
|
||||||
|
if conflict {
|
||||||
|
id = "first"
|
||||||
|
}
|
||||||
|
files := fstest.MapFS{"profiles.yaml": {Data: []byte(fmt.Sprintf("profiles:\n %s:\n backend: openrouter\n model: openai/gpt-5.6-luna\n", id))}}
|
||||||
|
if err := registry.RegisterFallbackProfileFS(files, "."); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
operator := filepath.Join(t.TempDir(), "operator.yml")
|
||||||
|
if err := os.WriteFile(operator, []byte("profiles:\n first:\n api_key_required: false\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: registry, Source: PromptKitProfileSourceConfig{ProfileFile: operator}})
|
||||||
|
if conflict {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("conflicting application profiles were silently overlaid")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"first", "second"} {
|
||||||
|
profile, err := inspector.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if profile.Model != "openai/gpt-5.6-luna" || profile.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("profile = %#v", profile)
|
||||||
|
}
|
||||||
|
if id == "first" && profile.CredentialRequired {
|
||||||
|
t.Fatal("operator overlay was not applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,12 +2,14 @@ package llm
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
|||||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, "prompts")
|
}, ".")
|
||||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, "root")
|
}, "root")
|
||||||
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||||
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
|||||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, ".")
|
}, ".")
|
||||||
|
|
||||||
engine := newPromptKitAssetTestEngine(t, registry)
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
if err != nil {
|
||||||
PromptID: "asset.test",
|
t.Fatal(err)
|
||||||
ProfileID: "asset-test-profile",
|
}
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
},
|
"asset-test-profile": {
|
||||||
})
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
Model: promptKitTestString("asset-test-model"),
|
||||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||||
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
|
||||||
|
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
|||||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, ".")
|
}, ".")
|
||||||
|
|
||||||
engine := newPromptKitAssetTestEngine(t, registry)
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"asset-test-profile": {
|
||||||
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
|
Model: promptKitTestString("asset-test-model"),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "asset.test",
|
PromptID: "asset.test",
|
||||||
ProfileID: "asset-test-profile",
|
ProfileID: "asset-test-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
|
||||||
|
|
||||||
_, err := registry.PromptFS()
|
_, err := registry.PromptFS()
|
||||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||||
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
|||||||
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
|
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
|
||||||
}, "first/profiles"); err != nil {
|
}, "first/profiles"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
|
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
|
||||||
}, "second"); err != nil {
|
}, "second"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
|
|||||||
|
|
||||||
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, err := registry.FallbackProfileFS()
|
_, err := registry.FallbackProfileFS()
|
||||||
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
|
|||||||
return digest
|
return digest
|
||||||
}
|
}
|
||||||
|
|
||||||
first := digestFor("id: fallback\nmodel: model-one\n")
|
first := digestFor("profiles:\n fallback:\n model: model-one\n")
|
||||||
second := digestFor("id: fallback\nmodel: model-two\n")
|
second := digestFor("profiles:\n fallback:\n model: model-two\n")
|
||||||
if first == second {
|
if first == second {
|
||||||
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
||||||
}
|
}
|
||||||
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
|||||||
|
|
||||||
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
options, err := registry.PromptKitOptions()
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
ID: "asset-test-profile",
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
"asset-test-profile": {
|
||||||
Model: "asset-test-model",
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
})))
|
Model: promptKitTestString("asset-test-model"),
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validPromptYAML(schemaPath string) string {
|
func validPromptYAML(schemaPath string) string {
|
||||||
return `id: asset.test
|
return `prompts:
|
||||||
version: "v1"
|
asset.test:
|
||||||
inputs:
|
v1:
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content_file: ./messages/user.tmpl
|
- role: user
|
||||||
- role: user
|
content_file: ./messages/user.tmpl
|
||||||
content_file: ./messages/task.tmpl
|
- role: user
|
||||||
output:
|
content_file: ./messages/task.tmpl
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: ` + schemaPath + `
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: ` + schemaPath + `
|
||||||
|
repair_attempts: 0
|
||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ type PromptKitLocalBackendConfig struct {
|
|||||||
type PromptKitClientConfig struct {
|
type PromptKitClientConfig struct {
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
ProfileFile string
|
ProfileFile string
|
||||||
|
Sources promptkit.ConfigurationSources
|
||||||
TransientRetries *int
|
TransientRetries *int
|
||||||
LocalBackend *PromptKitLocalBackendConfig
|
LocalBackend *PromptKitLocalBackendConfig
|
||||||
Assets *AssetRegistry
|
Assets *AssetRegistry
|
||||||
@@ -39,13 +40,10 @@ type PromptKitClientConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitClient struct {
|
type PromptKitClient struct {
|
||||||
engine *promptkit.Engine
|
engine *promptkit.Engine
|
||||||
recorder *LLMProfileRecorder
|
recorder *LLMProfileRecorder
|
||||||
profileDir string
|
checkpointFingerprints []CheckpointFingerprint
|
||||||
profileFile string
|
reasoningEffort *string
|
||||||
localEndpoint string
|
|
||||||
fallbackProfileDigest string
|
|
||||||
reasoningEffort *string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMProfileRecorder struct {
|
type LLMProfileRecorder struct {
|
||||||
@@ -53,40 +51,47 @@ type LLMProfileRecorder struct {
|
|||||||
profiles map[string]artifacts.LLMProfileManifest
|
profiles map[string]artifacts.LLMProfileManifest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptKitClientConfigurationError struct {
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *promptKitClientConfigurationError) Error() string {
|
||||||
|
return "PromptKit client configuration is invalid or unreadable"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *promptKitClientConfigurationError) Unwrap() error {
|
||||||
|
return e.err
|
||||||
|
}
|
||||||
|
|
||||||
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
|
var _ contracts.StructuredLLMClient = (*PromptKitClient)(nil)
|
||||||
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
|
var _ contracts.LLMProfileManifestProvider = (*PromptKitClient)(nil)
|
||||||
|
|
||||||
func PromptKitLocalBackendOption(cfg PromptKitLocalBackendConfig) promptkit.Option {
|
|
||||||
return promptkit.WithBackend(promptkit.LocalBackend(cfg.Endpoint, cfg.ConcurrencyLimit))
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||||
if cfg.Assets == nil {
|
if cfg.Assets == nil {
|
||||||
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
||||||
}
|
}
|
||||||
profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{
|
_, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
||||||
ProfileDir: cfg.ProfileDir,
|
ProfileDir: cfg.ProfileDir,
|
||||||
ProfileFile: cfg.ProfileFile,
|
ProfileFile: cfg.ProfileFile,
|
||||||
LocalBackend: cfg.LocalBackend,
|
LocalBackend: cfg.LocalBackend,
|
||||||
})
|
}, cfg.Assets, cfg.Sources)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
options, fallbackProfileDigest, err := cfg.Assets.promptKitOptions()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
options = append(options, profileOptions...)
|
|
||||||
options = append(options, cfg.EngineOptions...)
|
options = append(options, cfg.EngineOptions...)
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
ProfileDir: profileSource.ProfileDir,
|
Sources: sources,
|
||||||
Timeout: cfg.Timeout,
|
Timeout: cfg.Timeout,
|
||||||
HTTPClient: cfg.HTTPClient,
|
HTTPClient: cfg.HTTPClient,
|
||||||
TransientRetries: cfg.TransientRetries,
|
TransientRetries: cfg.TransientRetries,
|
||||||
}, options...)
|
}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create PromptKit engine: %w", err)
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
|
}
|
||||||
|
fingerprint, err := promptKitConfigurationFingerprint(engine)
|
||||||
|
if err != nil {
|
||||||
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
recorder := cfg.Recorder
|
recorder := cfg.Recorder
|
||||||
if recorder == nil {
|
if recorder == nil {
|
||||||
@@ -98,13 +103,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
reasoningEffort = &value
|
reasoningEffort = &value
|
||||||
}
|
}
|
||||||
return &PromptKitClient{
|
return &PromptKitClient{
|
||||||
engine: engine,
|
engine: engine,
|
||||||
recorder: recorder,
|
recorder: recorder,
|
||||||
profileDir: profileSource.ProfileDir,
|
checkpointFingerprints: []CheckpointFingerprint{fingerprint},
|
||||||
profileFile: profileSource.ProfileFile,
|
reasoningEffort: reasoningEffort,
|
||||||
localEndpoint: profileSource.localEndpoint(),
|
|
||||||
fallbackProfileDigest: fallbackProfileDigest,
|
|
||||||
reasoningEffort: reasoningEffort,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,15 +465,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
|
|||||||
if c == nil {
|
if c == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
|
return append([]CheckpointFingerprint(nil), c.checkpointFingerprints...), nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fingerprints := []CheckpointFingerprint{fingerprint}
|
|
||||||
if c.localEndpoint != "" {
|
|
||||||
fingerprints = append(fingerprints, promptKitLocalBackendFingerprint(c.localEndpoint))
|
|
||||||
}
|
|
||||||
return fingerprints, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
const promptKitConfigurationFingerprintName = "promptkit_configuration"
|
||||||
|
|
||||||
|
// This version describes the public definition graph captured by the engine.
|
||||||
|
// Include the release and catalog revisions because they contribute built-in
|
||||||
|
// profiles and other effective definitions without appearing in app sources.
|
||||||
|
const promptKitConfigurationSnapshotMarker = "promptkit:v0.12.0:openrouter:v3.20260930.0:rakestrawhome:v3.20260930.0:configuration-snapshot:v1"
|
||||||
|
|
||||||
|
type promptKitConfigurationFingerprintPayload struct {
|
||||||
|
SnapshotMarker string `json:"snapshot_marker"`
|
||||||
|
Configuration map[string]any `json:"configuration"`
|
||||||
|
Profiles []promptKitProfileFingerprintEntry `json:"profiles"`
|
||||||
|
Prompts []promptKitPromptFingerprintEntry `json:"prompts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptKitProfileFingerprintEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Target promptkit.ExecutionTarget `json:"target"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptKitPromptFingerprintEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
Hash string `json:"hash"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitConfigurationFingerprint(engine *promptkit.Engine) (CheckpointFingerprint, error) {
|
||||||
|
if engine == nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("PromptKit engine is unavailable for checkpoint identity")
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
var report bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&report); err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("write captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
var configuration map[string]any
|
||||||
|
if err := yaml.Unmarshal(report.Bytes(), &configuration); err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("decode captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
if configuration == nil {
|
||||||
|
configuration = map[string]any{}
|
||||||
|
}
|
||||||
|
removeLocalSchedulingFields(configuration)
|
||||||
|
|
||||||
|
payload := promptKitConfigurationFingerprintPayload{
|
||||||
|
SnapshotMarker: promptKitConfigurationSnapshotMarker,
|
||||||
|
Configuration: configuration,
|
||||||
|
}
|
||||||
|
profileIDs := sortedMapKeys(configuration["profiles"])
|
||||||
|
payload.Profiles = make([]promptKitProfileFingerprintEntry, 0, len(profileIDs))
|
||||||
|
for _, id := range profileIDs {
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit profile for checkpoint identity")
|
||||||
|
}
|
||||||
|
payload.Profiles = append(payload.Profiles, promptKitProfileFingerprintEntry{
|
||||||
|
ID: profile.ProfileID,
|
||||||
|
Target: profile.EffectiveModelParams,
|
||||||
|
APIKeyRequired: profile.APIKeyRequired,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
promptDefinitions, _ := configuration["prompts"].(map[string]any)
|
||||||
|
var promptIDs []string
|
||||||
|
for id := range promptDefinitions {
|
||||||
|
promptIDs = append(promptIDs, id)
|
||||||
|
}
|
||||||
|
sort.Strings(promptIDs)
|
||||||
|
for _, id := range promptIDs {
|
||||||
|
versions := sortedMapKeys(promptDefinitions[id])
|
||||||
|
for _, version := range versions {
|
||||||
|
prompt, err := engine.InspectPrompt(context.Background(), id, version)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit prompt for checkpoint identity")
|
||||||
|
}
|
||||||
|
payload.Prompts = append(payload.Prompts, promptKitPromptFingerprintEntry{
|
||||||
|
ID: prompt.PromptID,
|
||||||
|
Version: prompt.PromptVersion,
|
||||||
|
Hash: prompt.PromptHash,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("encode captured PromptKit checkpoint identity")
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(encoded)
|
||||||
|
return CheckpointFingerprint{
|
||||||
|
Name: promptKitConfigurationFingerprintName,
|
||||||
|
Value: "sha256:" + hex.EncodeToString(digest[:]),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedMapKeys(value any) []string {
|
||||||
|
values, _ := value.(map[string]any)
|
||||||
|
keys := make([]string, 0, len(values))
|
||||||
|
for key := range values {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeLocalSchedulingFields(configuration map[string]any) {
|
||||||
|
backends, _ := configuration["backends"].(map[string]any)
|
||||||
|
local, _ := backends["local"].(map[string]any)
|
||||||
|
delete(local, "concurrency_limit")
|
||||||
|
delete(local, "queue_capacity")
|
||||||
|
}
|
||||||
155
internal/framework/llm/promptkit_inspection.go
Normal file
155
internal/framework/llm/promptkit_inspection.go
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidPromptKitConfigurationPointer = errors.New("invalid PromptKit configuration pointer")
|
||||||
|
|
||||||
|
// PromptKitEffectiveModelParams is the provider-neutral JSON projection of an
|
||||||
|
// inspected profile target. Credential-shaped extra-parameter values are
|
||||||
|
// redacted in this reporting copy.
|
||||||
|
type PromptKitEffectiveModelParams struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
TopP float64 `json:"top_p"`
|
||||||
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
|
ServiceTier string `json:"service_tier"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type invalidPromptKitConfigurationPointerError struct {
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *invalidPromptKitConfigurationPointerError) Error() string {
|
||||||
|
return ErrInvalidPromptKitConfigurationPointer.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *invalidPromptKitConfigurationPointerError) Unwrap() error {
|
||||||
|
return e.cause
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *invalidPromptKitConfigurationPointerError) Is(target error) bool {
|
||||||
|
return target == ErrInvalidPromptKitConfigurationPointer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *PromptKitProfileInspector) EffectiveConfiguration() (string, error) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := i.WriteEffectiveConfiguration(&output); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return output.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *PromptKitProfileInspector) WriteEffectiveConfiguration(output io.Writer) error {
|
||||||
|
if i == nil || i.engine == nil {
|
||||||
|
return fmt.Errorf("PromptKit inspector must not be nil")
|
||||||
|
}
|
||||||
|
inspection, err := i.engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||||
|
}
|
||||||
|
return inspection.WriteEffective(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *PromptKitProfileInspector) ExplainConfiguration(pointer string) (string, bool, error) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
found, err := i.WriteConfigurationExplanation(&output, pointer)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
return output.String(), found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *PromptKitProfileInspector) WriteConfigurationExplanation(output io.Writer, pointer string) (bool, error) {
|
||||||
|
if i == nil || i.engine == nil {
|
||||||
|
return false, fmt.Errorf("PromptKit inspector must not be nil")
|
||||||
|
}
|
||||||
|
path, err := configkit.ParsePath(pointer)
|
||||||
|
if err != nil || path.String() != pointer {
|
||||||
|
if err == nil {
|
||||||
|
err = fmt.Errorf("pointer spelling is not canonical")
|
||||||
|
}
|
||||||
|
return false, &invalidPromptKitConfigurationPointerError{cause: err}
|
||||||
|
}
|
||||||
|
inspection, err := i.engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||||
|
}
|
||||||
|
if _, found := inspection.Explain(path); !found {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := inspection.WriteExplanation(output, path); err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitEffectiveModelParams(target promptkit.ExecutionTarget) PromptKitEffectiveModelParams {
|
||||||
|
return PromptKitEffectiveModelParams{
|
||||||
|
BackendID: strings.TrimSpace(target.BackendID),
|
||||||
|
Endpoint: strings.TrimSpace(target.Endpoint),
|
||||||
|
Model: strings.TrimSpace(target.Model),
|
||||||
|
Temperature: target.Temperature,
|
||||||
|
MaxTokens: target.MaxTokens,
|
||||||
|
TopP: target.TopP,
|
||||||
|
TimeoutSeconds: target.TimeoutSeconds,
|
||||||
|
ServiceTier: target.ServiceTier,
|
||||||
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
|
APIKeyEnv: strings.TrimSpace(target.APIKeyEnv),
|
||||||
|
ExtraParams: redactCredentialParameters(target.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func redactCredentialParameters(values map[string]any) map[string]any {
|
||||||
|
if values == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
redacted, _ := redactCredentialParameterValue(values).(map[string]any)
|
||||||
|
return redacted
|
||||||
|
}
|
||||||
|
|
||||||
|
func redactCredentialParameterValue(value any) any {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
copy := make(map[string]any, len(typed))
|
||||||
|
for key, nested := range typed {
|
||||||
|
if isCredentialParameterName(key) {
|
||||||
|
copy[key] = "[REDACTED]"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
copy[key] = redactCredentialParameterValue(nested)
|
||||||
|
}
|
||||||
|
return copy
|
||||||
|
case []any:
|
||||||
|
copy := make([]any, len(typed))
|
||||||
|
for index, nested := range typed {
|
||||||
|
copy[index] = redactCredentialParameterValue(nested)
|
||||||
|
}
|
||||||
|
return copy
|
||||||
|
default:
|
||||||
|
return typed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isCredentialParameterName(name string) bool {
|
||||||
|
switch strings.ToLower(name) {
|
||||||
|
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
143
internal/framework/llm/promptkit_inspection_test.go
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type promptKitInspectionErrorWriter struct{ err error }
|
||||||
|
|
||||||
|
func (w promptKitInspectionErrorWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||||
|
|
||||||
|
func TestPromptKitProfileInspectionCopiesAndRedactsTarget(t *testing.T) {
|
||||||
|
const (
|
||||||
|
profileID = "inspection/profile~case"
|
||||||
|
keyEnv = "NOT_SET_SYNTHETIC_PROFILE_KEY"
|
||||||
|
)
|
||||||
|
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||||
|
params := map[string]any{
|
||||||
|
"Authorization": "Bearer authorization-secret",
|
||||||
|
"api_key": "api-key-secret",
|
||||||
|
"ApiKey": "api-key-camel-secret",
|
||||||
|
"apikey": "apikey-secret",
|
||||||
|
"credential": "credential-secret",
|
||||||
|
"credentials": "credentials-secret",
|
||||||
|
"password": "password-secret",
|
||||||
|
"secret": "secret-secret",
|
||||||
|
"token": "token-secret",
|
||||||
|
"access_token": "access-token-secret",
|
||||||
|
"normal": "visible-value",
|
||||||
|
"nested": map[string]any{
|
||||||
|
"Access_Token": "nested-token-secret",
|
||||||
|
"items": []any{map[string]any{"PASSWORD": "list-secret", "label": "ordinary"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||||
|
profileID: {Endpoint: &endpoint, Model: &initialModel, APIKeyEnv: promptKitTestString(keyEnv), ExtraParams: params},
|
||||||
|
})}}},
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||||
|
profileID: {Model: &finalModel},
|
||||||
|
})}}},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||||
|
}
|
||||||
|
profile, err := inspector.InspectProfile(t.Context(), profileID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InspectProfile() error = %v; an unset synthetic credential name must not be resolved", err)
|
||||||
|
}
|
||||||
|
if profile.ProfileID != profileID || profile.BackendID != "" || profile.Model != finalModel || profile.CredentialEnvironment != keyEnv || profile.CredentialRequired {
|
||||||
|
t.Fatalf("profile inspection = %#v", profile)
|
||||||
|
}
|
||||||
|
paramsCopy := profile.EffectiveModelParams.ExtraParams
|
||||||
|
for _, key := range []string{"Authorization", "api_key", "ApiKey", "apikey", "credential", "credentials", "password", "secret", "token", "access_token"} {
|
||||||
|
if paramsCopy[key] != "[REDACTED]" {
|
||||||
|
t.Errorf("effective provider parameter %q = %#v, want redacted", key, paramsCopy[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if paramsCopy["normal"] != "visible-value" {
|
||||||
|
t.Fatalf("top-level effective provider parameters = %#v", paramsCopy)
|
||||||
|
}
|
||||||
|
nested := paramsCopy["nested"].(map[string]any)
|
||||||
|
if nested["Access_Token"] != "[REDACTED]" {
|
||||||
|
t.Fatalf("nested effective provider parameters = %#v", nested)
|
||||||
|
}
|
||||||
|
items := nested["items"].([]any)
|
||||||
|
if items[0].(map[string]any)["PASSWORD"] != "[REDACTED]" || items[0].(map[string]any)["label"] != "ordinary" {
|
||||||
|
t.Fatalf("list provider parameters = %#v", items)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(profile.EffectiveModelParams)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, secret := range []string{"authorization-secret", "api-key-secret", "api-key-camel-secret", "apikey-secret", "credential-secret", "credentials-secret", "password-secret", "secret-secret", "token-secret", "access-token-secret", "nested-token-secret", "list-secret"} {
|
||||||
|
if strings.Contains(string(encoded), secret) {
|
||||||
|
t.Fatalf("profile reporting projection contains %q", secret)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nested["ordinary"] = "caller mutation"
|
||||||
|
paramsCopy["normal"] = "caller mutation"
|
||||||
|
second, err := inspector.InspectProfile(t.Context(), profileID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if second.EffectiveModelParams.ExtraParams["normal"] != "visible-value" || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["ordinary"] != nil {
|
||||||
|
t.Fatalf("profile inspection retained caller mutation: %#v", second.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitConfigurationInspectionDistinguishesInvalidAndUnknownPointers(t *testing.T) {
|
||||||
|
const profileID = "inspection/profile~case"
|
||||||
|
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
|
||||||
|
profileID: {Endpoint: &endpoint, Model: &initialModel},
|
||||||
|
})}}},
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
|
||||||
|
profileID: {Model: &finalModel},
|
||||||
|
})}}},
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := inspector.EffectiveConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EffectiveConfiguration() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(report, "model-after") || strings.Contains(report, "model-before") {
|
||||||
|
t.Fatalf("effective report = %q, want only the effective model", report)
|
||||||
|
}
|
||||||
|
const escapedPointer = "/profiles/inspection~1profile~0case/model"
|
||||||
|
explanation, found, err := inspector.ExplainConfiguration(escapedPointer)
|
||||||
|
if err != nil || !found || !strings.Contains(explanation, "model-before") || !strings.Contains(explanation, "model-after") {
|
||||||
|
t.Fatalf("ExplainConfiguration(%q) = (%q, %t, %v), want effective and historical values", escapedPointer, explanation, found, err)
|
||||||
|
}
|
||||||
|
if _, found, err := inspector.ExplainConfiguration("/profiles/not-configured"); err != nil || found {
|
||||||
|
t.Fatalf("unknown configuration pointer = (found %t, error %v), want (false, nil)", found, err)
|
||||||
|
}
|
||||||
|
if _, _, err := inspector.ExplainConfiguration("/profiles/~2/model"); !errors.Is(err, ErrInvalidPromptKitConfigurationPointer) {
|
||||||
|
t.Fatalf("invalid configuration pointer error = %v", err)
|
||||||
|
}
|
||||||
|
if _, found, err := inspector.ExplainConfiguration("/profiles/inspection~01profile~00case/model"); err != nil || found {
|
||||||
|
t.Fatalf("configuration pointer with literal escape sequences = (found %t, error %v), want a valid unknown pointer", found, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
writerFailure := errors.New("writer failure")
|
||||||
|
if err := inspector.WriteEffectiveConfiguration(promptKitInspectionErrorWriter{err: writerFailure}); !errors.Is(err, writerFailure) {
|
||||||
|
t.Fatalf("WriteEffectiveConfiguration() error = %v, want writer cause", err)
|
||||||
|
}
|
||||||
|
if _, err := inspector.WriteConfigurationExplanation(promptKitInspectionErrorWriter{err: writerFailure}, escapedPointer); !errors.Is(err, writerFailure) {
|
||||||
|
t.Fatalf("WriteConfigurationExplanation() error = %v, want writer cause", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,9 +23,9 @@ func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int,
|
|||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
|
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
|
||||||
ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model",
|
Endpoint: promptKitTestString(server.URL + "/v1"), Model: promptKitTestString("test-model"),
|
||||||
}))},
|
}}),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
promptKitProfileFingerprintName = "promptkit_profile_source"
|
|
||||||
promptKitLocalBackendFingerprintName = "promptkit_local_backend_target"
|
|
||||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
|
||||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
|
||||||
// release. Update this identity when the dependency is upgraded.
|
|
||||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.11.2:builtin-profiles"
|
|
||||||
)
|
|
||||||
|
|
||||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
|
||||||
hasher := sha256.New()
|
|
||||||
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
|
|
||||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case strings.TrimSpace(profileFile) != "":
|
|
||||||
cleanProfileFile := strings.TrimSpace(profileFile)
|
|
||||||
data, err := os.ReadFile(cleanProfileFile)
|
|
||||||
if err != nil {
|
|
||||||
return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
|
|
||||||
}
|
|
||||||
writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
|
|
||||||
writeFingerprintPart(hasher, data)
|
|
||||||
case strings.TrimSpace(profileDir) != "":
|
|
||||||
files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
|
|
||||||
if err != nil {
|
|
||||||
return CheckpointFingerprint{}, err
|
|
||||||
}
|
|
||||||
for _, file := range files {
|
|
||||||
writeFingerprintPart(hasher, []byte(file.path))
|
|
||||||
writeFingerprintPart(hasher, file.digest)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return CheckpointFingerprint{
|
|
||||||
Name: promptKitProfileFingerprintName,
|
|
||||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
|
|
||||||
hasher := sha256.New()
|
|
||||||
writeFingerprintPart(hasher, []byte(promptKitLocalBackendMarker))
|
|
||||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(endpoint)))
|
|
||||||
return CheckpointFingerprint{
|
|
||||||
Name: promptKitLocalBackendFingerprintName,
|
|
||||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptKitProfileFile struct {
|
|
||||||
path string
|
|
||||||
digest []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
|
|
||||||
var files []promptKitProfileFile
|
|
||||||
err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
|
|
||||||
if walkErr != nil {
|
|
||||||
return walkErr
|
|
||||||
}
|
|
||||||
if entry.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
extension := filepath.Ext(entry.Name())
|
|
||||||
if extension != ".yaml" && extension != ".yml" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
data, err := os.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
relativePath, err := filepath.Rel(root, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(data)
|
|
||||||
files = append(files, promptKitProfileFile{
|
|
||||||
path: filepath.ToSlash(relativePath),
|
|
||||||
digest: append([]byte(nil), sum[:]...),
|
|
||||||
})
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
|
|
||||||
}
|
|
||||||
sort.Slice(files, func(i, j int) bool {
|
|
||||||
return files[i].path < files[j].path
|
|
||||||
})
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {
|
|
||||||
length := []byte(fmt.Sprintf("%d:", len(value)))
|
|
||||||
_, _ = hasher.Write(length)
|
|
||||||
_, _ = hasher.Write(value)
|
|
||||||
}
|
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,8 +17,9 @@ type PromptKitProfileSourceConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitProfileInspectorConfig struct {
|
type PromptKitProfileInspectorConfig struct {
|
||||||
Source PromptKitProfileSourceConfig
|
Source PromptKitProfileSourceConfig
|
||||||
Assets *AssetRegistry
|
Assets *AssetRegistry
|
||||||
|
Sources promptkit.ConfigurationSources
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
||||||
@@ -37,6 +39,7 @@ type PromptKitProfileInspection struct {
|
|||||||
Model string
|
Model string
|
||||||
CredentialEnvironment string
|
CredentialEnvironment string
|
||||||
CredentialRequired bool
|
CredentialRequired bool
|
||||||
|
EffectiveModelParams PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitProfileInspectionError struct {
|
type PromptKitProfileInspectionError struct {
|
||||||
@@ -74,22 +77,12 @@ func (e *promptKitProfileConfigurationError) Unwrap() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
||||||
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
|
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets, cfg.Sources)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, &promptKitProfileConfigurationError{err: err}
|
||||||
}
|
|
||||||
if cfg.Assets != nil {
|
|
||||||
fallbackOption, hasFallback, err := cfg.Assets.promptKitFallbackProfileOption()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if hasFallback {
|
|
||||||
options = append(options, fallbackOption)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: ".",
|
Sources: sources,
|
||||||
ProfileDir: source.ProfileDir,
|
|
||||||
}, options...)
|
}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &promptKitProfileConfigurationError{err: err}
|
return nil, &promptKitProfileConfigurationError{err: err}
|
||||||
@@ -101,7 +94,6 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
|||||||
if i == nil || i.engine == nil {
|
if i == nil || i.engine == nil {
|
||||||
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
|
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
|
||||||
}
|
}
|
||||||
profileID = strings.TrimSpace(profileID)
|
|
||||||
inspection, err := i.engine.InspectProfile(ctx, profileID)
|
inspection, err := i.engine.InspectProfile(ctx, profileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
@@ -118,16 +110,38 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
|||||||
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
|
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
|
||||||
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
|
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
|
||||||
CredentialRequired: inspection.APIKeyRequired,
|
CredentialRequired: inspection.APIKeyRequired,
|
||||||
|
EffectiveModelParams: promptKitEffectiveModelParams(inspection.EffectiveModelParams),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) {
|
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry, configuredSources promptkit.ConfigurationSources) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
|
||||||
|
profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
||||||
|
}
|
||||||
|
sources := promptkit.ConfigurationSources{}
|
||||||
|
var options []promptkit.Option
|
||||||
|
if assets != nil {
|
||||||
|
maintainedSources, maintainedOptions, err := assets.PromptKitConfigurationSources()
|
||||||
|
if err != nil {
|
||||||
|
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
||||||
|
}
|
||||||
|
sources = maintainedSources
|
||||||
|
options = append(options, maintainedOptions...)
|
||||||
|
}
|
||||||
|
sources.Underlays = append(sources.Underlays, configuredSources.Underlays...)
|
||||||
|
sources.Underlays = append(sources.Underlays, profileSources.Underlays...)
|
||||||
|
sources.Main = configuredSources.Main
|
||||||
|
return profileSource, sources, options, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitProfileConfigurationSources(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, error) {
|
||||||
source := PromptKitProfileSourceConfig{
|
source := PromptKitProfileSourceConfig{
|
||||||
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
|
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
|
||||||
ProfileFile: strings.TrimSpace(cfg.ProfileFile),
|
ProfileFile: strings.TrimSpace(cfg.ProfileFile),
|
||||||
}
|
}
|
||||||
if source.ProfileDir != "" && source.ProfileFile != "" {
|
if source.ProfileDir != "" && source.ProfileFile != "" {
|
||||||
return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
|
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
|
||||||
}
|
}
|
||||||
if cfg.LocalBackend != nil {
|
if cfg.LocalBackend != nil {
|
||||||
localBackend := *cfg.LocalBackend
|
localBackend := *cfg.LocalBackend
|
||||||
@@ -135,12 +149,31 @@ func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (Prom
|
|||||||
source.LocalBackend = &localBackend
|
source.LocalBackend = &localBackend
|
||||||
}
|
}
|
||||||
|
|
||||||
var options []promptkit.Option
|
var underlays []configkit.Underlay
|
||||||
if source.ProfileFile != "" {
|
if source.ProfileFile != "" {
|
||||||
options = append(options, promptkit.WithProfileFile(source.ProfileFile))
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
|
Source: configkit.File(source.ProfileFile),
|
||||||
|
Imports: true,
|
||||||
|
}}})
|
||||||
|
} else if source.ProfileDir != "" {
|
||||||
|
inputs, err := promptkit.DirectoryInputs(source.ProfileDir)
|
||||||
|
if err != nil {
|
||||||
|
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, fmt.Errorf("select PromptKit profile directory: %w", err)
|
||||||
|
}
|
||||||
|
for index := range inputs {
|
||||||
|
inputs[index].Imports = true
|
||||||
|
}
|
||||||
|
if len(inputs) != 0 {
|
||||||
|
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if source.LocalBackend != nil {
|
if source.LocalBackend != nil {
|
||||||
options = append(options, PromptKitLocalBackendOption(*source.LocalBackend))
|
backend := promptkit.LocalBackend(source.LocalBackend.Endpoint, source.LocalBackend.ConcurrencyLimit)
|
||||||
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.BackendSource("notarius-local-backend", map[string]promptkit.BackendConfig{
|
||||||
|
promptkit.BackendLocal: backend,
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
}
|
}
|
||||||
return source, options, nil
|
return source, promptkit.ConfigurationSources{Underlays: underlays}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
462
internal/framework/llm/promptkit_sources_integration_test.go
Normal file
462
internal/framework/llm/promptkit_sources_integration_test.go
Normal file
@@ -0,0 +1,462 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) {
|
||||||
|
for _, name := range []string{
|
||||||
|
"NOTARIUS_TEST_BACKEND_KEY",
|
||||||
|
"NOTARIUS_TEST_PROFILE_KEY",
|
||||||
|
"NOTARIUS_TEST_CATALOG_KEY",
|
||||||
|
} {
|
||||||
|
t.Setenv(name, "")
|
||||||
|
}
|
||||||
|
operatorDir := t.TempDir()
|
||||||
|
profilePath := filepath.Join(operatorDir, "operator.yaml")
|
||||||
|
if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles:
|
||||||
|
inherited-base:
|
||||||
|
backend: custom-backend
|
||||||
|
model: imported-model
|
||||||
|
temperature: 0.7
|
||||||
|
service_tier: flex
|
||||||
|
extra_params:
|
||||||
|
inherited: retained
|
||||||
|
nested:
|
||||||
|
inherited: true
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(profilePath, []byte(`configkit:
|
||||||
|
imports:
|
||||||
|
- path: inherited.yaml
|
||||||
|
backends:
|
||||||
|
custom-backend:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
api_key_env: NOTARIUS_TEST_BACKEND_KEY
|
||||||
|
concurrency_limit: 1
|
||||||
|
queue_capacity: 0
|
||||||
|
openrouter:
|
||||||
|
api_key_env: NOTARIUS_TEST_CATALOG_KEY
|
||||||
|
profiles:
|
||||||
|
custom-profile:
|
||||||
|
extends: inherited-base
|
||||||
|
model: operator-model
|
||||||
|
temperature: 0
|
||||||
|
api_key_required: false
|
||||||
|
service_tier: null
|
||||||
|
extra_params: null
|
||||||
|
empty-map-profile:
|
||||||
|
extends: inherited-base
|
||||||
|
extra_params: {}
|
||||||
|
api_key_required: false
|
||||||
|
profile-key-profile:
|
||||||
|
backend: custom-backend
|
||||||
|
model: profile-key-model
|
||||||
|
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
||||||
|
api_key_required: false
|
||||||
|
required-profile:
|
||||||
|
extends: inherited-base
|
||||||
|
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
||||||
|
api_key_required: true
|
||||||
|
catalog-profile:
|
||||||
|
backend: openrouter
|
||||||
|
model: openai/gpt-5.6-luna
|
||||||
|
api_key_required: false
|
||||||
|
prompts:
|
||||||
|
adapter.test:
|
||||||
|
v1:
|
||||||
|
output:
|
||||||
|
repair_attempts: 1
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assets := newTestPromptKitAssets(t)
|
||||||
|
fake := &fakePromptKitLLM{}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: assets,
|
||||||
|
ProfileFile: profilePath,
|
||||||
|
EngineOptions: []promptkit.Option{
|
||||||
|
promptkit.WithLLMClient(fake),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||||
|
}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||||
|
Source: PromptKitProfileSourceConfig{ProfileFile: profilePath},
|
||||||
|
Assets: assets,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
id string
|
||||||
|
backend string
|
||||||
|
model string
|
||||||
|
keyEnv string
|
||||||
|
required bool
|
||||||
|
}{
|
||||||
|
{id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||||
|
{id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||||
|
{id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
||||||
|
{id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"},
|
||||||
|
{id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true},
|
||||||
|
{id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"},
|
||||||
|
} {
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), test.id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InspectProfile(%q) error = %v", test.id, err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model ||
|
||||||
|
inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required {
|
||||||
|
t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var out map[string]any
|
||||||
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
ProfileID: "custom-profile",
|
||||||
|
SessionID: "source-composition-test",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err)
|
||||||
|
}
|
||||||
|
if response.Debug == nil || response.Debug.Prompt == nil {
|
||||||
|
t.Fatalf("completion debug = %#v, want captured prompt", response.Debug)
|
||||||
|
}
|
||||||
|
debug := response.Debug.Prompt
|
||||||
|
if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" {
|
||||||
|
t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID)
|
||||||
|
}
|
||||||
|
if got := debug.EffectiveModelParams["model"]; got != "operator-model" {
|
||||||
|
t.Fatalf("runtime model = %#v, want operator-model", got)
|
||||||
|
}
|
||||||
|
if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 {
|
||||||
|
t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"])
|
||||||
|
}
|
||||||
|
if got := debug.EffectiveModelParams["service_tier"]; got == "flex" {
|
||||||
|
t.Fatalf("explicit null retained inherited service tier: %#v", got)
|
||||||
|
}
|
||||||
|
if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil {
|
||||||
|
t.Fatalf("explicit null retained inherited provider map: %#v", got)
|
||||||
|
}
|
||||||
|
if got := debug.OutputContract["repair_attempts"]; got != float64(1) {
|
||||||
|
t.Fatalf("overlaid output repair attempts = %#v, want 1", got)
|
||||||
|
}
|
||||||
|
if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") {
|
||||||
|
t.Fatalf("rendered prompt messages = %#v", debug.Messages)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
||||||
|
t.Fatalf("provider calls after optional-credential execution = %d, want 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
ProfileID: "required-profile",
|
||||||
|
SessionID: "source-composition-test",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") {
|
||||||
|
t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
||||||
|
t.Fatalf("provider calls after required-credential failure = %d, want 1", got)
|
||||||
|
}
|
||||||
|
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.test",
|
||||||
|
ProfileID: "empty-map-profile",
|
||||||
|
SessionID: "source-composition-test",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteStructured() with empty overlay map error = %v", err)
|
||||||
|
}
|
||||||
|
if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" {
|
||||||
|
t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) {
|
||||||
|
rootDirectory := t.TempDir()
|
||||||
|
declaringDirectory := filepath.Join(rootDirectory, "child")
|
||||||
|
replacementDirectory := t.TempDir()
|
||||||
|
writeFile := func(path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rootConfig := filepath.Join(rootDirectory, "operator.yaml")
|
||||||
|
declaringConfig := filepath.Join(declaringDirectory, "imported.yaml")
|
||||||
|
replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml")
|
||||||
|
writeFile(rootConfig, `configkit:
|
||||||
|
imports:
|
||||||
|
- path: child/imported.yaml
|
||||||
|
`)
|
||||||
|
writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy")
|
||||||
|
writeFile(declaringConfig, `prompts:
|
||||||
|
imported.capture:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: messages/shared.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import")
|
||||||
|
writeFile(replacementConfig, `prompts:
|
||||||
|
adapter.test:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: messages/shared.tmpl
|
||||||
|
`)
|
||||||
|
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original")
|
||||||
|
|
||||||
|
endpoint, model := "https://example.invalid/v1", "content-context-model"
|
||||||
|
newClient := func() *PromptKitClient {
|
||||||
|
t.Helper()
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: newTestPromptKitAssets(t),
|
||||||
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"content-context-profile": {Endpoint: &endpoint, Model: &model},
|
||||||
|
})}}},
|
||||||
|
{Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}},
|
||||||
|
{Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
prepareMessage := func(client *PromptKitClient, promptID string) string {
|
||||||
|
t.Helper()
|
||||||
|
request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"}
|
||||||
|
if promptID == "adapter.test" {
|
||||||
|
request.Inputs = map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline(`{"source":true}`),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prepared, err := client.engine.Prepare(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Prepare(%q) error = %v", promptID, err)
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 {
|
||||||
|
t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages)
|
||||||
|
}
|
||||||
|
return prepared.Messages[0].Content
|
||||||
|
}
|
||||||
|
|
||||||
|
first := newClient()
|
||||||
|
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||||
|
t.Fatalf("imported prompt content = %q, want declaring-source content", got)
|
||||||
|
}
|
||||||
|
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||||
|
t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got)
|
||||||
|
}
|
||||||
|
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated")
|
||||||
|
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated")
|
||||||
|
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||||
|
t.Fatalf("existing engine imported prompt content after edit = %q", got)
|
||||||
|
}
|
||||||
|
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||||
|
t.Fatalf("existing engine replacement content after edit = %q", got)
|
||||||
|
}
|
||||||
|
second := newClient()
|
||||||
|
if got := prepareMessage(second, "imported.capture"); got != "declared import updated" {
|
||||||
|
t.Fatalf("new engine imported prompt content = %q", got)
|
||||||
|
}
|
||||||
|
if got := prepareMessage(second, "adapter.test"); got != "replacement updated" {
|
||||||
|
t.Fatalf("new engine replacement content = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
prompt string
|
||||||
|
files fstest.MapFS
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing message file",
|
||||||
|
prompt: `prompts:
|
||||||
|
construction.failure:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: messages/missing.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid static template",
|
||||||
|
prompt: `prompts:
|
||||||
|
construction.failure:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: '{{ .request_value'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
assets := NewAssetRegistry()
|
||||||
|
files := test.files
|
||||||
|
if files == nil {
|
||||||
|
files = fstest.MapFS{}
|
||||||
|
}
|
||||||
|
files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)}
|
||||||
|
if err := assets.RegisterPromptFS(files, "."); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: assets,
|
||||||
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": {
|
||||||
|
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||||
|
Model: promptKitTestString("construction-model"),
|
||||||
|
}}),
|
||||||
|
})
|
||||||
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) {
|
||||||
|
assets := NewAssetRegistry()
|
||||||
|
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||||
|
"prompt.yaml": {Data: []byte(`prompts:
|
||||||
|
request.variable:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Value: {{ .request_value }}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
}, "."); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: assets,
|
||||||
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": {
|
||||||
|
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||||
|
Model: promptKitTestString("request-model"),
|
||||||
|
}}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err)
|
||||||
|
}
|
||||||
|
if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "request.variable", ProfileID: "request-profile",
|
||||||
|
}); !errors.Is(err, promptkit.ErrPromptRender) {
|
||||||
|
t.Fatalf("Prepare() error = %v, want missing request variable render failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) {
|
||||||
|
t.Run("directory group conflicts", func(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} {
|
||||||
|
content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||||
|
Source: PromptKitProfileSourceConfig{ProfileDir: directory},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) {
|
||||||
|
badEndpoint := "not an endpoint"
|
||||||
|
goodEndpoint := "http://127.0.0.1:1/v1"
|
||||||
|
lowerModel := "lower-model"
|
||||||
|
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{
|
||||||
|
"layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel},
|
||||||
|
})}}},
|
||||||
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{
|
||||||
|
"layered-profile": {Endpoint: &goodEndpoint},
|
||||||
|
})}}},
|
||||||
|
}}
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err)
|
||||||
|
}
|
||||||
|
inspection, err := inspector.InspectProfile(context.Background(), "layered-profile")
|
||||||
|
if err != nil || inspection.Model != lowerModel {
|
||||||
|
t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty and missing selected directories", func(t *testing.T) {
|
||||||
|
if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||||
|
Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")},
|
||||||
|
}); err == nil {
|
||||||
|
t.Fatal("missing selected directory was accepted")
|
||||||
|
}
|
||||||
|
empty := t.TempDir()
|
||||||
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
||||||
|
Source: PromptKitProfileSourceConfig{ProfileDir: empty},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("empty selected directory error = %v, want no contribution", err)
|
||||||
|
}
|
||||||
|
if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil {
|
||||||
|
t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unused invalid final profile fails construction", func(t *testing.T) {
|
||||||
|
badEndpoint := "not an endpoint"
|
||||||
|
model := "unused-model"
|
||||||
|
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"unused-profile": {Endpoint: &badEndpoint, Model: &model},
|
||||||
|
}),
|
||||||
|
}}}}}
|
||||||
|
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
||||||
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func promptKitTestString(value string) *string { return &value }
|
||||||
|
|
||||||
|
func promptKitTestInt(value int) *int { return &value }
|
||||||
|
|
||||||
|
func promptKitTestFloat(value float64) *float64 { return &value }
|
||||||
|
|
||||||
|
func promptKitTestInputs(t *testing.T, label string, files fstest.MapFS, root string) []configkit.Input {
|
||||||
|
t.Helper()
|
||||||
|
inputs, err := promptkit.FilesystemInputs(label, files, root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FilesystemInputs() error = %v", err)
|
||||||
|
}
|
||||||
|
return inputs
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitTestSources(
|
||||||
|
backends map[string]promptkit.BackendConfig,
|
||||||
|
profiles map[string]promptkit.ProfileConfig,
|
||||||
|
otherInputs ...configkit.Input,
|
||||||
|
) promptkit.ConfigurationSources {
|
||||||
|
inputs := make([]configkit.Input, 0, len(otherInputs)+2)
|
||||||
|
if len(backends) != 0 {
|
||||||
|
inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("test backends", backends)})
|
||||||
|
}
|
||||||
|
if len(profiles) != 0 {
|
||||||
|
inputs = append(inputs, configkit.Input{Source: promptkit.ProfileSource("test profiles", profiles)})
|
||||||
|
}
|
||||||
|
inputs = append(inputs, otherInputs...)
|
||||||
|
if len(inputs) == 0 {
|
||||||
|
return promptkit.ConfigurationSources{}
|
||||||
|
}
|
||||||
|
return promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -17,17 +18,9 @@ func TestRegisterAssetsPreparesGenericPromptOffline(t *testing.T) {
|
|||||||
if err := RegisterAssets(registry); err != nil {
|
if err := RegisterAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterAssets() error = %v, want nil", err)
|
t.Fatalf("RegisterAssets() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"semantic-reconciliation-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "offline-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "semantic-reconciliation-test", Endpoint: "http://127.0.0.1:1/v1", Model: "offline-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
|
PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "semantic-reconciliation-test",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -131,18 +132,7 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party
|
|||||||
|
|
||||||
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
func newScenesPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
options, err := registry.PromptKitOptions()
|
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"scene-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "scene-test-profile",
|
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
|
||||||
Model: "scene-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
return engine
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,17 +35,9 @@ func TestPromptAssetsPrepareRequiredInputs(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"combat-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
transcript := `{"units":[{"sentinel":"combat-transcript"}]}`
|
transcript := `{"units":[{"sentinel":"combat-transcript"}]}`
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -63,18 +64,9 @@ func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"),
|
||||||
t.Fatal(err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "dnd-extraction", Endpoint: "http://127.0.0.1:1/v1", Model: "enemy-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return engine
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,17 +17,9 @@ func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
|
||||||
t.Fatal(err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "item-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-occurrences-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"item-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-test-model"})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||||
}})
|
}})
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,17 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||||
t.Fatal(err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "location-occurrences-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,15 +16,9 @@ func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
t.Fatalf("RegisterPromptAssets() error = %v", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "location-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "location-test-model"})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||||
}})
|
}})
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,17 +24,9 @@ func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) {
|
|||||||
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
|
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
|
||||||
t.Fatalf("response schema asset: %v", err)
|
t.Fatalf("response schema asset: %v", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"),
|
||||||
t.Fatal(err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "npc-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-occurrences-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
|
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"npc-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,17 +17,9 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -88,19 +89,9 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party
|
|||||||
if err := RegisterPromptAssets(registry); err != nil {
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
t.Fatalf("register spell prompt assets: %v", err)
|
t.Fatalf("register spell prompt assets: %v", err)
|
||||||
}
|
}
|
||||||
options, err := registry.PromptKitOptions()
|
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||||
if err != nil {
|
"spell-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "spell-test-model"),
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
})
|
||||||
}
|
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
||||||
ID: "spell-test-profile",
|
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
|
||||||
Model: "spell-test-model",
|
|
||||||
})))
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
||||||
}
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: PromptID,
|
PromptID: PromptID,
|
||||||
PromptVersion: SchemaVersion,
|
PromptVersion: SchemaVersion,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user