feat(ai): add prompt_version to ai_usage + wire version constants
Migration 000024 adds prompt_version column + partial index. PromptVersion plumbed through ChatRequest -> UsageEvent -> buildUsageEvent -> settings INSERT/SELECT. Version constants defined in ai/versions.go and wired at all three call sites.
This commit is contained in:
@@ -90,12 +90,13 @@ func (e *ProviderLLMEnricher) EnrichMissing(ctx context.Context, req LLMRequest)
|
||||
userPrompt := buildUserPrompt(req, blocks)
|
||||
|
||||
resp, err := e.AI.Chat(ctx, &ai.ChatRequest{
|
||||
SystemPrompt: systemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONSchema: enricherSchemaJSON,
|
||||
Grounded: true,
|
||||
CallType: "enrich_b",
|
||||
Temperature: 0.1,
|
||||
SystemPrompt: systemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONSchema: enricherSchemaJSON,
|
||||
Grounded: true,
|
||||
CallType: "enrich_b",
|
||||
PromptVersion: ai.VersionEnrichB,
|
||||
Temperature: 0.1,
|
||||
})
|
||||
if err != nil {
|
||||
return Enrichment{}, fmt.Errorf("chat: %w", err)
|
||||
|
||||
@@ -114,12 +114,13 @@ func (c *SimilarityClassifierLLM) Classify(ctx context.Context, a, b SimilarityR
|
||||
userPrompt := simUserPrompt(a, b)
|
||||
|
||||
resp, err := c.AI.Chat(ctx, &ai.ChatRequest{
|
||||
SystemPrompt: systemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONMode: true,
|
||||
Grounded: false,
|
||||
CallType: "similarity",
|
||||
Temperature: 0.1,
|
||||
SystemPrompt: systemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONMode: true,
|
||||
Grounded: false,
|
||||
CallType: "similarity",
|
||||
PromptVersion: ai.VersionSimilarity,
|
||||
Temperature: 0.1,
|
||||
})
|
||||
if err != nil {
|
||||
return Verdict{}, fmt.Errorf("chat: %w", err)
|
||||
|
||||
@@ -125,11 +125,12 @@ func (o *Orchestrator) Run(ctx context.Context, in Input) (Output, error) {
|
||||
|
||||
func callLLM(ctx context.Context, p ai.Provider, userPrompt string, schema []byte) (*ai.ChatResponse, error) {
|
||||
return p.Chat(ctx, &ai.ChatRequest{
|
||||
SystemPrompt: SystemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONSchema: schema,
|
||||
CallType: "research",
|
||||
Temperature: 0.1,
|
||||
SystemPrompt: SystemPrompt,
|
||||
UserMessage: userPrompt,
|
||||
JSONSchema: schema,
|
||||
CallType: "research",
|
||||
PromptVersion: ai.VersionResearch,
|
||||
Temperature: 0.1,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -25,13 +25,17 @@ func (r *UsageRepo) Record(ctx context.Context, e ai.UsageEvent) error {
|
||||
if e.Error != "" {
|
||||
errStr = &e.Error
|
||||
}
|
||||
var promptVersion *string
|
||||
if e.PromptVersion != "" {
|
||||
promptVersion = &e.PromptVersion
|
||||
}
|
||||
_, err := r.db.Exec(ctx, `
|
||||
INSERT INTO ai_usage
|
||||
(provider, model, call_type, input_tokens, output_tokens,
|
||||
grounded, duration_ms, estimated_cost_usd, error)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
grounded, duration_ms, estimated_cost_usd, error, prompt_version)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
`, e.Provider, e.Model, e.CallType, e.InputTokens, e.OutputTokens,
|
||||
e.Grounded, e.DurationMs, e.EstimatedCostUSD, errStr)
|
||||
e.Grounded, e.DurationMs, e.EstimatedCostUSD, errStr, promptVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("usage: record: %w", err)
|
||||
}
|
||||
@@ -95,13 +99,14 @@ type UsageEvent struct {
|
||||
DurationMs int `json:"duration_ms"`
|
||||
EstimatedCostUSD float64 `json:"estimated_cost_usd"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||
}
|
||||
|
||||
func (r *UsageRepo) Recent(ctx context.Context, limit int) ([]UsageEvent, error) {
|
||||
rows, err := r.db.Query(ctx, `
|
||||
SELECT id, created_at, provider, model, call_type,
|
||||
input_tokens, output_tokens, grounded, duration_ms,
|
||||
estimated_cost_usd, error
|
||||
estimated_cost_usd, error, prompt_version
|
||||
FROM ai_usage
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $1
|
||||
@@ -116,7 +121,7 @@ func (r *UsageRepo) Recent(ctx context.Context, limit int) ([]UsageEvent, error)
|
||||
var e UsageEvent
|
||||
if err := rows.Scan(&e.ID, &e.CreatedAt, &e.Provider, &e.Model, &e.CallType,
|
||||
&e.InputTokens, &e.OutputTokens, &e.Grounded, &e.DurationMs,
|
||||
&e.EstimatedCostUSD, &e.Error); err != nil {
|
||||
&e.EstimatedCostUSD, &e.Error, &e.PromptVersion); err != nil {
|
||||
return nil, fmt.Errorf("usage: scan: %w", err)
|
||||
}
|
||||
out = append(out, e)
|
||||
|
||||
@@ -285,11 +285,12 @@ func (p *GeminiProvider) Chat(ctx context.Context, req *ChatRequest) (*ChatRespo
|
||||
|
||||
func (p *GeminiProvider) buildUsageEvent(model string, req *ChatRequest, resp *genai.GenerateContentResponse, callErr error, durationMs int) UsageEvent {
|
||||
e := UsageEvent{
|
||||
Provider: "gemini",
|
||||
Model: model,
|
||||
CallType: req.CallType,
|
||||
Grounded: req.Grounded,
|
||||
DurationMs: durationMs,
|
||||
Provider: "gemini",
|
||||
Model: model,
|
||||
CallType: req.CallType,
|
||||
PromptVersion: req.PromptVersion,
|
||||
Grounded: req.Grounded,
|
||||
DurationMs: durationMs,
|
||||
}
|
||||
if callErr != nil {
|
||||
e.Error = callErr.Error()
|
||||
|
||||
16
backend/internal/pkg/ai/prompt_version_test.go
Normal file
16
backend/internal/pkg/ai/prompt_version_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package ai
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBuildUsageEvent_PropagatesPromptVersion(t *testing.T) {
|
||||
p := &GeminiProvider{}
|
||||
req := &ChatRequest{
|
||||
CallType: "test",
|
||||
PromptVersion: "test/v1",
|
||||
Grounded: false,
|
||||
}
|
||||
e := p.buildUsageEvent("gemini-test", req, nil, nil, 42)
|
||||
if e.PromptVersion != "test/v1" {
|
||||
t.Errorf("PromptVersion = %q; want %q", e.PromptVersion, "test/v1")
|
||||
}
|
||||
}
|
||||
@@ -21,15 +21,16 @@ type ModelSelector interface {
|
||||
}
|
||||
|
||||
type ChatRequest struct {
|
||||
SystemPrompt string
|
||||
UserMessage string
|
||||
Model string
|
||||
MaxTokens int
|
||||
Temperature float32
|
||||
JSONMode bool
|
||||
JSONSchema json.RawMessage
|
||||
Grounded bool // request Google Search grounding (Gemini only)
|
||||
CallType string // e.g. "research", "enrich_b", "similarity" — for usage tracking
|
||||
SystemPrompt string
|
||||
UserMessage string
|
||||
Model string
|
||||
MaxTokens int
|
||||
Temperature float32
|
||||
JSONMode bool
|
||||
JSONSchema json.RawMessage
|
||||
Grounded bool // request Google Search grounding (Gemini only)
|
||||
CallType string // e.g. "research", "enrich_b", "similarity" — for usage tracking
|
||||
PromptVersion string // e.g. "research/v2" — for correlation with ai_usage
|
||||
}
|
||||
|
||||
type ChatResponse struct {
|
||||
|
||||
@@ -13,6 +13,7 @@ type UsageEvent struct {
|
||||
DurationMs int
|
||||
EstimatedCostUSD float64
|
||||
Error string // empty on success
|
||||
PromptVersion string
|
||||
}
|
||||
|
||||
// UsageRecorder persists a UsageEvent. Implementations must be safe for
|
||||
|
||||
9
backend/internal/pkg/ai/versions.go
Normal file
9
backend/internal/pkg/ai/versions.go
Normal file
@@ -0,0 +1,9 @@
|
||||
package ai
|
||||
|
||||
// Prompt version constants. Bump when the system prompt for a call type
|
||||
// changes materially so ai_usage rows can be correlated with prompt iterations.
|
||||
const (
|
||||
VersionResearch = "research/v2"
|
||||
VersionEnrichB = "enrich_b/v2"
|
||||
VersionSimilarity = "similarity/v2"
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS ai_usage_prompt_version_idx;
|
||||
ALTER TABLE ai_usage DROP COLUMN IF EXISTS prompt_version;
|
||||
4
backend/migrations/000024_ai_usage_prompt_version.up.sql
Normal file
4
backend/migrations/000024_ai_usage_prompt_version.up.sql
Normal file
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE ai_usage ADD COLUMN prompt_version TEXT NULL;
|
||||
CREATE INDEX ai_usage_prompt_version_idx
|
||||
ON ai_usage (prompt_version, created_at DESC)
|
||||
WHERE prompt_version IS NOT NULL;
|
||||
Reference in New Issue
Block a user