Update examples and benchmarks for identified resources
This commit is contained in:
@@ -84,13 +84,15 @@ func run() (summary, error) {
|
||||
return summary{}, err
|
||||
}
|
||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||
backend.ID = backendID
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
backend,
|
||||
})},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
|
||||
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
|
||||
})},
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.assessment:
|
||||
version: "1.0.0"
|
||||
- id: example.assessment
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.proposal:
|
||||
version: "1.0.0"
|
||||
- id: example.proposal
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
backends:
|
||||
offline:
|
||||
- id: offline
|
||||
endpoint: https://review.example.invalid/v1
|
||||
profiles:
|
||||
audit:
|
||||
- id: audit
|
||||
backend: review/offline
|
||||
model: review-model
|
||||
prompts:
|
||||
check:
|
||||
- id: check
|
||||
default_profile: review/audit
|
||||
description: Inspect a review item without contacting a model provider.
|
||||
messages:
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
version: "1"
|
||||
version: "2"
|
||||
config:
|
||||
imports:
|
||||
- path: fragments/resources.yaml
|
||||
required: true
|
||||
- path: fragments/prepare.yaml
|
||||
destination: [prompts]
|
||||
placement: sequence-item
|
||||
required: true
|
||||
release_version: "1.4.0"
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
id: prepare
|
||||
default_profile: support/fast
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content_file: ../messages/note.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -1,28 +1,11 @@
|
||||
backends:
|
||||
offline:
|
||||
- id: offline
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
base:
|
||||
- id: base
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
timeout_seconds: 90
|
||||
fast:
|
||||
extends: base
|
||||
- id: fast
|
||||
extends: {id: base}
|
||||
timeout_seconds: 60
|
||||
prompts:
|
||||
prepare:
|
||||
version: "1.0.0"
|
||||
default_profile: support/fast
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content_file: ../messages/note.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# A later input intentionally replaces only the inherited profile timeout.
|
||||
# A later input replaces the profile with an inherited variant.
|
||||
profiles:
|
||||
fast:
|
||||
- id: fast
|
||||
extends: {id: base}
|
||||
timeout_seconds: 30
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
type summary struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
@@ -70,9 +69,7 @@ func main() {
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys(
|
||||
"collections", "support", "profiles", "fast", "timeout_seconds",
|
||||
)); !found {
|
||||
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||
exit(fmt.Errorf("collection profile override was not found"))
|
||||
}
|
||||
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
@@ -148,7 +145,6 @@ func main() {
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(summary{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
SelectedProfile: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
|
||||
@@ -41,8 +41,8 @@ type summary struct {
|
||||
func main() {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
prompts:
|
||||
example.run:
|
||||
version: "1.0.0"
|
||||
- id: example.run
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
|
||||
Reference in New Issue
Block a user