Use native request execution and credential masking
This commit is contained in:
@@ -5,15 +5,15 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// CallHooks receive one logical start and finish record and the execution
|
||||
@@ -25,22 +25,29 @@ type CallHooks struct {
|
||||
}
|
||||
|
||||
type RoleClient struct {
|
||||
runtime *Runtime
|
||||
role string
|
||||
hooks CallHooks
|
||||
runtime *Runtime
|
||||
role string
|
||||
profileID string
|
||||
settings config.LLM
|
||||
hooks CallHooks
|
||||
credentialMu sync.Mutex
|
||||
credentialNames map[string]struct{}
|
||||
}
|
||||
|
||||
var _ structuredoutput.Client = (*RoleClient)(nil)
|
||||
|
||||
func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, error) {
|
||||
if _, ok := r.Profiles[role]; !ok {
|
||||
return nil, fmt.Errorf("unknown LLM role %q", role)
|
||||
func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hooks CallHooks) (*RoleClient, error) {
|
||||
if r == nil || r.Engine == nil {
|
||||
return nil, fmt.Errorf("LLM engine is unavailable")
|
||||
}
|
||||
return &RoleClient{runtime: r, role: role, hooks: hooks}, nil
|
||||
if profileID == "" {
|
||||
return nil, fmt.Errorf("profile selection is required")
|
||||
}
|
||||
return &RoleClient{runtime: r, role: role, profileID: profileID, settings: settings, hooks: hooks}, nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
|
||||
ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout)
|
||||
ctx, cancel := context.WithTimeout(parent, c.settings.OperationTimeout)
|
||||
defer cancel()
|
||||
var terminalMu sync.Mutex
|
||||
var terminal pk.ExecutionEvent
|
||||
@@ -64,8 +71,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
}()
|
||||
record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()}
|
||||
record.Context.Role = c.role
|
||||
profile := c.runtime.Profiles[c.role]
|
||||
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: profile.ProfileID, BackendID: profile.EffectiveModelParams.BackendID, Model: profile.EffectiveModelParams.Model}
|
||||
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
|
||||
if c.hooks.OnCall != nil {
|
||||
c.hooks.OnCall(record)
|
||||
}
|
||||
@@ -89,26 +95,15 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
if err = validateOutputTarget(out); err != nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}
|
||||
}
|
||||
schemaPath, ok := responseschema.AssetPath(req.Schema)
|
||||
if !ok {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
|
||||
}
|
||||
backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID]
|
||||
vars := make(map[string]string, len(req.Variables)+1)
|
||||
for key, value := range req.Variables {
|
||||
vars[key] = value
|
||||
}
|
||||
for key, value := range prompts.SharedVariables() {
|
||||
vars[key] = value
|
||||
}
|
||||
apiKey := ""
|
||||
if backend.Credential != nil {
|
||||
apiKey = backend.Credential.Value()
|
||||
contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts)
|
||||
if contractErr != nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
|
||||
}
|
||||
vars := prompts.WithSharedVariables(req.Variables)
|
||||
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars,
|
||||
Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts},
|
||||
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
|
||||
TransientRetries: &c.settings.TransientRetries,
|
||||
Validation: contract,
|
||||
Observer: func(event pk.ExecutionEvent) {
|
||||
if event.Kind == pk.ExecutionFinished {
|
||||
terminalMu.Lock()
|
||||
@@ -129,16 +124,19 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
}
|
||||
defer handle.Discard()
|
||||
details := handle.Details()
|
||||
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
|
||||
c.credentialMu.Lock()
|
||||
if c.credentialNames == nil {
|
||||
c.credentialNames = make(map[string]struct{})
|
||||
}
|
||||
c.credentialNames[name] = struct{}{}
|
||||
c.credentialMu.Unlock()
|
||||
}
|
||||
response.Prompt = promptMetadata(details)
|
||||
record.Prompt = response.Prompt
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
response.Prepared = preparedSnapshot(details)
|
||||
record.Prepared = response.Prepared
|
||||
selected, exists := c.runtime.Config.Backends[details.SelectedBackendID]
|
||||
endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") }
|
||||
if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
|
||||
}
|
||||
result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
@@ -169,6 +167,15 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// OutputContract maps an Audita domain schema to the native request contract.
|
||||
func OutputContract(schema responseschema.Key, repairs int) (*pk.OutputContract, error) {
|
||||
path, ok := responseschema.AssetPath(schema)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("schema %q", schema)
|
||||
}
|
||||
return &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: path, RepairAttempts: repairs}, nil
|
||||
}
|
||||
|
||||
// classifiedFailure deliberately checks setup defects before broad PromptKit
|
||||
// validation/generation wrappers. Provider text is kept only in the typed cause.
|
||||
func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
|
||||
@@ -239,7 +246,7 @@ func isSetupFailure(err error) bool {
|
||||
|
||||
func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
||||
return structuredoutput.PromptMetadata{
|
||||
PromptID: details.PromptID, PromptVersion: details.PromptVersion,
|
||||
PromptID: details.PromptID,
|
||||
DefinitionHash: details.PromptHash, RenderedHash: details.RenderedPromptHash,
|
||||
ProfileID: details.SelectedProfileID, BackendID: details.SelectedBackendID,
|
||||
Model: details.EffectiveModelParams.Model,
|
||||
@@ -253,3 +260,14 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot
|
||||
}
|
||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
||||
}
|
||||
|
||||
// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials.
|
||||
func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
|
||||
c.credentialMu.Lock()
|
||||
names := make([]string, 0, len(c.credentialNames))
|
||||
for name := range c.credentialNames {
|
||||
names = append(names, name)
|
||||
}
|
||||
c.credentialMu.Unlock()
|
||||
return c.runtime.ConfiguredSecrets(ctx, names...)
|
||||
}
|
||||
|
||||
286
internal/framework/llm/native_adapter_test.go
Normal file
286
internal/framework/llm/native_adapter_test.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type transportFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
|
||||
t.Setenv("AUDITA_TEST_KEY", "synthetic-secret-value")
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
document := "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: test-model\n api_key_env: AUDITA_TEST_KEY\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(document), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
providerErr := errors.New("provider echoed synthetic-secret-value")
|
||||
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" {
|
||||
t.Errorf("authorization missing")
|
||||
}
|
||||
return nil, providerErr
|
||||
})}
|
||||
runtime, err := NewRuntime(context.Background(), loaded, httpClient)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", "fixture", config.LLM{OperationTimeout: time.Second}, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}, &out)
|
||||
if err == nil || calls != 1 {
|
||||
t.Fatalf("error = %v, calls = %d", err, calls)
|
||||
}
|
||||
masked := runtime.RedactError(context.Background(), providerErr)
|
||||
if strings.Contains(masked, "synthetic-secret-value") {
|
||||
t.Fatal("credential not redacted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, body := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: test-model\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
repair int
|
||||
outputs []string
|
||||
wantCalls int
|
||||
success bool
|
||||
}{
|
||||
{"success", 0, []string{`{"corrections":[]}`}, 1, true},
|
||||
{"repair", 1, []string{`{"wrong":true}`, `{"corrections":[]}`}, 2, true},
|
||||
{"no repair", 0, []string{`{"wrong":true}`}, 1, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
item := tc.outputs[min(calls, len(tc.outputs)-1)]
|
||||
calls++
|
||||
payload, _ := json.Marshal(item)
|
||||
body := fmt.Sprintf(`{"model":"test-model","choices":[{"message":{"content":%s}}]}`, payload)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})}
|
||||
runtime, err := NewRuntime(context.Background(), loaded, httpClient)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", "fixture", config.LLM{OperationTimeout: 10 * time.Second, RepairAttempts: tc.repair}, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), request, &result)
|
||||
if (err == nil) != tc.success || calls != tc.wantCalls {
|
||||
t.Fatalf("error=%v calls=%d", err, calls)
|
||||
}
|
||||
if response.Prompt.ProfileID != "fixture" || response.Prompt.Model != "test-model" {
|
||||
t.Fatalf("prepared target = %+v", response.Prompt)
|
||||
}
|
||||
if !tc.success && !structuredoutput.IsRecoverable(err) {
|
||||
t.Fatalf("expected recoverable failure: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
runtime, err := NewRuntime(context.Background(), loaded, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", "fixture", config.LLM{OperationTimeout: time.Second}, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.CompleteStructured(ctx, request, &map[string]any{})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleClientsShareBackendCapacity(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, body := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "backends:\n - id: shared\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: first\n backend: shared\n model: one\n - id: second\n backend: shared\n model: two\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
body := `{"model":"one","choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := config.LLM{OperationTimeout: 10 * time.Second}
|
||||
first, err := runtime.NewRoleClient("proposal", "first", settings, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := runtime.NewRoleClient("validation", "second", settings, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := first.CompleteStructured(context.Background(), request, &map[string]any{})
|
||||
done <- err
|
||||
}()
|
||||
<-entered
|
||||
_, secondErr := second.CompleteStructured(context.Background(), request, &map[string]any{})
|
||||
close(release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(secondErr, &failure) || failure.Reason != outcomes.ReasonCapacityExceeded {
|
||||
t.Fatalf("second call = %v", secondErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleClientRetriesFollowRequestBudget(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: test-model\n"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
status, body := 503, `{"error":{"message":"unavailable"}}`
|
||||
if calls > 1 {
|
||||
status, body = 200, `{"model":"test-model","choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`
|
||||
}
|
||||
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", "fixture", config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: 1}, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}
|
||||
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil || calls != 2 || response.Execution.TransientRetries != 1 {
|
||||
t.Fatalf("error=%v calls=%d execution=%+v", err, calls, response.Execution)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
|
||||
t.Setenv("AUDITA_BACKEND_KEY", "backend-secret-value")
|
||||
t.Setenv("AUDITA_PROFILE_KEY", "profile-secret-value")
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "backends:\n - id: service\n endpoint: https://example.invalid/v1\n api_key_env: AUDITA_BACKEND_KEY\nprofiles:\n - id: inherited\n backend: service\n model: fixture\n api_key_env: null\n - id: explicit\n backend: service\n model: fixture\n api_key_env: AUDITA_PROFILE_KEY\n - id: cleared\n backend: service\n model: fixture\n api_key_env: ''\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
authorization := ""
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
authorization = r.Header.Get("Authorization")
|
||||
body := `{"model":"fixture","choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}
|
||||
for _, tc := range []struct{ profile, auth string }{{"inherited", "Bearer backend-secret-value"}, {"explicit", "Bearer profile-secret-value"}, {"cleared", ""}} {
|
||||
t.Run(tc.profile, func(t *testing.T) {
|
||||
client, err := runtime.NewRoleClient("proposal", tc.profile, config.LLM{OperationTimeout: 5 * time.Second}, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if authorization != tc.auth {
|
||||
t.Fatalf("authorization = %q", authorization)
|
||||
}
|
||||
snapshot, err := json.Marshal(response.Prepared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(snapshot), "secret-value") {
|
||||
t.Fatal("prepared snapshot contains credential")
|
||||
}
|
||||
})
|
||||
}
|
||||
secrets := runtime.ConfiguredSecrets(context.Background())
|
||||
for _, value := range []string{"backend-secret-value", "profile-secret-value"} {
|
||||
if !slices.Contains(secrets, value) {
|
||||
t.Fatalf("missing credential masking value")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// TokenCount retains the distinction between zero and unavailable usage.
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
@@ -120,10 +119,3 @@ func embeddedAssetOrInputError(ctx context.Context, inputs []ck.Input, cause err
|
||||
}
|
||||
|
||||
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
|
||||
|
||||
func RedactError(err error, c config.Config) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
||||
}
|
||||
|
||||
@@ -4,58 +4,11 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type selectedProfileView struct {
|
||||
ID string `yaml:"id"`
|
||||
Backend string `yaml:"backend"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier,omitempty"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort,omitempty"`
|
||||
}
|
||||
|
||||
// SelectedProfilesYAML exposes selected controls without endpoint, credential,
|
||||
// or arbitrary extra provider parameters.
|
||||
func (r *Runtime) SelectedProfilesYAML() ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("LLM runtime is unavailable")
|
||||
}
|
||||
view := func(role string) (selectedProfileView, error) {
|
||||
profile, ok := r.Profiles[role]
|
||||
if !ok {
|
||||
return selectedProfileView{}, fmt.Errorf("%s profile is unavailable", role)
|
||||
}
|
||||
target := profile.EffectiveModelParams
|
||||
return selectedProfileView{ID: profile.ProfileID, Backend: target.BackendID, Model: target.Model,
|
||||
Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP,
|
||||
TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort}, nil
|
||||
}
|
||||
proposal, err := view("proposal")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
validation, err := view("validation")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := yaml.Marshal(struct {
|
||||
Proposal selectedProfileView `yaml:"proposal"`
|
||||
Validation selectedProfileView `yaml:"validation"`
|
||||
}{proposal, validation})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return redaction.New(ConfiguredSecrets(&r.Config)).YAML(raw)
|
||||
}
|
||||
|
||||
type derivationView struct {
|
||||
Operation ck.ChangeKind `yaml:"operation"`
|
||||
Reason string `yaml:"reason"`
|
||||
|
||||
@@ -1,18 +1,54 @@
|
||||
package llm
|
||||
|
||||
import "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
// ConfiguredSecrets returns all configured LLM API-key values that should be
|
||||
// redacted from diagnostics and surfaced error payloads.
|
||||
func ConfiguredSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
)
|
||||
|
||||
// ConfiguredSecrets obtains only values referenced by captured credential metadata.
|
||||
// It never selects a credential or treats an absent value as an error.
|
||||
func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string {
|
||||
if r == nil || r.Engine == nil {
|
||||
return nil
|
||||
}
|
||||
secrets := make([]string, 0, len(cfg.Backends))
|
||||
for _, backend := range cfg.Backends {
|
||||
if backend.Credential != nil && backend.Credential.Value() != "" {
|
||||
secrets = append(secrets, backend.Credential.Value())
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
names := make(map[string]struct{})
|
||||
add := func(name *string) {
|
||||
if name != nil && *name != "" {
|
||||
names[*name] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, name := range additionalEnvNames {
|
||||
add(&name)
|
||||
}
|
||||
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
|
||||
for _, id := range ids {
|
||||
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
|
||||
add(profile.APIKeyEnv)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ids, err := r.Engine.ListBackends(ctx); err == nil {
|
||||
for _, id := range ids {
|
||||
if backend, err := r.Engine.InspectBackend(ctx, id); err == nil {
|
||||
add(backend.APIKeyEnv)
|
||||
}
|
||||
}
|
||||
}
|
||||
secrets := make([]string, 0, len(names))
|
||||
for name := range names {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
secrets = append(secrets, value)
|
||||
}
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error())
|
||||
}
|
||||
|
||||
@@ -60,7 +60,6 @@ type Request struct {
|
||||
Glossary *schema.Glossary `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
@@ -105,10 +104,16 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
var secrets []string
|
||||
if source, ok := req.LLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = source.ConfiguredSecrets(ctx)
|
||||
}
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||
llm.ConfiguredSecrets(req.Config),
|
||||
secrets,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -120,7 +125,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
artifacts InteractionArtifacts
|
||||
)
|
||||
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
PromptID: req.PromptID,
|
||||
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
|
||||
}, &response)
|
||||
|
||||
@@ -63,7 +63,6 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
Glossary: req.ProposalRequest.Glossary,
|
||||
Config: req.ProposalRequest.Config,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
PromptVersion: promptMetadata.PromptVersion,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
|
||||
@@ -22,17 +22,15 @@ type DiagnosticContext struct {
|
||||
// Request names an embedded prompt and response schema. Profile selection and
|
||||
// credentials are supplied by the role-bound runtime client.
|
||||
type Request struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
Schema responseschema.Key `json:"schema"`
|
||||
Context DiagnosticContext `json:"context"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
Schema responseschema.Key `json:"schema"`
|
||||
Context DiagnosticContext `json:"context"`
|
||||
}
|
||||
|
||||
// PromptMetadata is the execution snapshot returned by the runtime when available.
|
||||
type PromptMetadata struct {
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
DefinitionHash string `json:"definition_hash,omitempty"`
|
||||
RenderedHash string `json:"rendered_hash,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
|
||||
@@ -148,7 +148,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
|
||||
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
||||
PromptID: promptMetadata.PromptID, PromptVersion: promptMetadata.PromptVersion,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
|
||||
}, &response)
|
||||
|
||||
@@ -82,6 +82,20 @@ func SharedVariables() map[string]string {
|
||||
}
|
||||
}
|
||||
|
||||
// WithSharedVariables fills missing Audita instruction variables without replacing caller values.
|
||||
func WithSharedVariables(values map[string]string) map[string]string {
|
||||
result := make(map[string]string, len(values)+3)
|
||||
for key, value := range values {
|
||||
result[key] = value
|
||||
}
|
||||
for key, value := range SharedVariables() {
|
||||
if _, exists := result[key]; !exists {
|
||||
result[key] = value
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func HardeningText() string { return sharedText("prompt_hardening.md") }
|
||||
|
||||
func sharedText(name string) string {
|
||||
|
||||
Reference in New Issue
Block a user