From ad1da8be66b5b199250c4b1efc29277da232cb25 Mon Sep 17 00:00:00 2001 From: vikingowl Date: Sat, 25 Apr 2026 18:08:53 +0200 Subject: [PATCH] feat(ai): add prompt_version to ai_usage + wire version constants Migration 000024 adds prompt_version column + partial index. PromptVersion plumbed through ChatRequest -> UsageEvent -> buildUsageEvent -> settings INSERT/SELECT. Version constants defined in ai/versions.go and wired at all three call sites. --- .../domain/discovery/enrich/llm_enricher.go | 13 +++++++------ .../domain/discovery/enrich/similarity.go | 13 +++++++------ .../domain/market/research/orchestrator.go | 11 ++++++----- backend/internal/domain/settings/usage.go | 15 ++++++++++----- backend/internal/pkg/ai/gemini.go | 11 ++++++----- .../internal/pkg/ai/prompt_version_test.go | 16 ++++++++++++++++ backend/internal/pkg/ai/provider.go | 19 ++++++++++--------- backend/internal/pkg/ai/usage.go | 1 + backend/internal/pkg/ai/versions.go | 9 +++++++++ .../000024_ai_usage_prompt_version.down.sql | 2 ++ .../000024_ai_usage_prompt_version.up.sql | 4 ++++ 11 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 backend/internal/pkg/ai/prompt_version_test.go create mode 100644 backend/internal/pkg/ai/versions.go create mode 100644 backend/migrations/000024_ai_usage_prompt_version.down.sql create mode 100644 backend/migrations/000024_ai_usage_prompt_version.up.sql diff --git a/backend/internal/domain/discovery/enrich/llm_enricher.go b/backend/internal/domain/discovery/enrich/llm_enricher.go index d8a7f1d..7adc5c7 100644 --- a/backend/internal/domain/discovery/enrich/llm_enricher.go +++ b/backend/internal/domain/discovery/enrich/llm_enricher.go @@ -90,12 +90,13 @@ func (e *ProviderLLMEnricher) EnrichMissing(ctx context.Context, req LLMRequest) userPrompt := buildUserPrompt(req, blocks) resp, err := e.AI.Chat(ctx, &ai.ChatRequest{ - SystemPrompt: systemPrompt, - UserMessage: userPrompt, - JSONSchema: enricherSchemaJSON, - Grounded: true, - CallType: "enrich_b", - Temperature: 0.1, + SystemPrompt: systemPrompt, + UserMessage: userPrompt, + JSONSchema: enricherSchemaJSON, + Grounded: true, + CallType: "enrich_b", + PromptVersion: ai.VersionEnrichB, + Temperature: 0.1, }) if err != nil { return Enrichment{}, fmt.Errorf("chat: %w", err) diff --git a/backend/internal/domain/discovery/enrich/similarity.go b/backend/internal/domain/discovery/enrich/similarity.go index d62a95d..6c894fe 100644 --- a/backend/internal/domain/discovery/enrich/similarity.go +++ b/backend/internal/domain/discovery/enrich/similarity.go @@ -114,12 +114,13 @@ func (c *SimilarityClassifierLLM) Classify(ctx context.Context, a, b SimilarityR userPrompt := simUserPrompt(a, b) resp, err := c.AI.Chat(ctx, &ai.ChatRequest{ - SystemPrompt: systemPrompt, - UserMessage: userPrompt, - JSONMode: true, - Grounded: false, - CallType: "similarity", - Temperature: 0.1, + SystemPrompt: systemPrompt, + UserMessage: userPrompt, + JSONMode: true, + Grounded: false, + CallType: "similarity", + PromptVersion: ai.VersionSimilarity, + Temperature: 0.1, }) if err != nil { return Verdict{}, fmt.Errorf("chat: %w", err) diff --git a/backend/internal/domain/market/research/orchestrator.go b/backend/internal/domain/market/research/orchestrator.go index 86b3b99..14f9816 100644 --- a/backend/internal/domain/market/research/orchestrator.go +++ b/backend/internal/domain/market/research/orchestrator.go @@ -125,11 +125,12 @@ func (o *Orchestrator) Run(ctx context.Context, in Input) (Output, error) { func callLLM(ctx context.Context, p ai.Provider, userPrompt string, schema []byte) (*ai.ChatResponse, error) { return p.Chat(ctx, &ai.ChatRequest{ - SystemPrompt: SystemPrompt, - UserMessage: userPrompt, - JSONSchema: schema, - CallType: "research", - Temperature: 0.1, + SystemPrompt: SystemPrompt, + UserMessage: userPrompt, + JSONSchema: schema, + CallType: "research", + PromptVersion: ai.VersionResearch, + Temperature: 0.1, }) } diff --git a/backend/internal/domain/settings/usage.go b/backend/internal/domain/settings/usage.go index 347b61f..9939ecd 100644 --- a/backend/internal/domain/settings/usage.go +++ b/backend/internal/domain/settings/usage.go @@ -25,13 +25,17 @@ func (r *UsageRepo) Record(ctx context.Context, e ai.UsageEvent) error { if e.Error != "" { errStr = &e.Error } + var promptVersion *string + if e.PromptVersion != "" { + promptVersion = &e.PromptVersion + } _, err := r.db.Exec(ctx, ` INSERT INTO ai_usage (provider, model, call_type, input_tokens, output_tokens, - grounded, duration_ms, estimated_cost_usd, error) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + grounded, duration_ms, estimated_cost_usd, error, prompt_version) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) `, e.Provider, e.Model, e.CallType, e.InputTokens, e.OutputTokens, - e.Grounded, e.DurationMs, e.EstimatedCostUSD, errStr) + e.Grounded, e.DurationMs, e.EstimatedCostUSD, errStr, promptVersion) if err != nil { return fmt.Errorf("usage: record: %w", err) } @@ -95,13 +99,14 @@ type UsageEvent struct { DurationMs int `json:"duration_ms"` EstimatedCostUSD float64 `json:"estimated_cost_usd"` Error *string `json:"error,omitempty"` + PromptVersion *string `json:"prompt_version,omitempty"` } func (r *UsageRepo) Recent(ctx context.Context, limit int) ([]UsageEvent, error) { rows, err := r.db.Query(ctx, ` SELECT id, created_at, provider, model, call_type, input_tokens, output_tokens, grounded, duration_ms, - estimated_cost_usd, error + estimated_cost_usd, error, prompt_version FROM ai_usage ORDER BY created_at DESC LIMIT $1 @@ -116,7 +121,7 @@ func (r *UsageRepo) Recent(ctx context.Context, limit int) ([]UsageEvent, error) var e UsageEvent if err := rows.Scan(&e.ID, &e.CreatedAt, &e.Provider, &e.Model, &e.CallType, &e.InputTokens, &e.OutputTokens, &e.Grounded, &e.DurationMs, - &e.EstimatedCostUSD, &e.Error); err != nil { + &e.EstimatedCostUSD, &e.Error, &e.PromptVersion); err != nil { return nil, fmt.Errorf("usage: scan: %w", err) } out = append(out, e) diff --git a/backend/internal/pkg/ai/gemini.go b/backend/internal/pkg/ai/gemini.go index fdfa28c..e872069 100644 --- a/backend/internal/pkg/ai/gemini.go +++ b/backend/internal/pkg/ai/gemini.go @@ -285,11 +285,12 @@ func (p *GeminiProvider) Chat(ctx context.Context, req *ChatRequest) (*ChatRespo func (p *GeminiProvider) buildUsageEvent(model string, req *ChatRequest, resp *genai.GenerateContentResponse, callErr error, durationMs int) UsageEvent { e := UsageEvent{ - Provider: "gemini", - Model: model, - CallType: req.CallType, - Grounded: req.Grounded, - DurationMs: durationMs, + Provider: "gemini", + Model: model, + CallType: req.CallType, + PromptVersion: req.PromptVersion, + Grounded: req.Grounded, + DurationMs: durationMs, } if callErr != nil { e.Error = callErr.Error() diff --git a/backend/internal/pkg/ai/prompt_version_test.go b/backend/internal/pkg/ai/prompt_version_test.go new file mode 100644 index 0000000..4f3992f --- /dev/null +++ b/backend/internal/pkg/ai/prompt_version_test.go @@ -0,0 +1,16 @@ +package ai + +import "testing" + +func TestBuildUsageEvent_PropagatesPromptVersion(t *testing.T) { + p := &GeminiProvider{} + req := &ChatRequest{ + CallType: "test", + PromptVersion: "test/v1", + Grounded: false, + } + e := p.buildUsageEvent("gemini-test", req, nil, nil, 42) + if e.PromptVersion != "test/v1" { + t.Errorf("PromptVersion = %q; want %q", e.PromptVersion, "test/v1") + } +} diff --git a/backend/internal/pkg/ai/provider.go b/backend/internal/pkg/ai/provider.go index c35c210..15f2c17 100644 --- a/backend/internal/pkg/ai/provider.go +++ b/backend/internal/pkg/ai/provider.go @@ -21,15 +21,16 @@ type ModelSelector interface { } type ChatRequest struct { - SystemPrompt string - UserMessage string - Model string - MaxTokens int - Temperature float32 - JSONMode bool - JSONSchema json.RawMessage - Grounded bool // request Google Search grounding (Gemini only) - CallType string // e.g. "research", "enrich_b", "similarity" — for usage tracking + SystemPrompt string + UserMessage string + Model string + MaxTokens int + Temperature float32 + JSONMode bool + JSONSchema json.RawMessage + Grounded bool // request Google Search grounding (Gemini only) + CallType string // e.g. "research", "enrich_b", "similarity" — for usage tracking + PromptVersion string // e.g. "research/v2" — for correlation with ai_usage } type ChatResponse struct { diff --git a/backend/internal/pkg/ai/usage.go b/backend/internal/pkg/ai/usage.go index 4a47ff1..b540d86 100644 --- a/backend/internal/pkg/ai/usage.go +++ b/backend/internal/pkg/ai/usage.go @@ -13,6 +13,7 @@ type UsageEvent struct { DurationMs int EstimatedCostUSD float64 Error string // empty on success + PromptVersion string } // UsageRecorder persists a UsageEvent. Implementations must be safe for diff --git a/backend/internal/pkg/ai/versions.go b/backend/internal/pkg/ai/versions.go new file mode 100644 index 0000000..4cc2fe6 --- /dev/null +++ b/backend/internal/pkg/ai/versions.go @@ -0,0 +1,9 @@ +package ai + +// Prompt version constants. Bump when the system prompt for a call type +// changes materially so ai_usage rows can be correlated with prompt iterations. +const ( + VersionResearch = "research/v2" + VersionEnrichB = "enrich_b/v2" + VersionSimilarity = "similarity/v2" +) diff --git a/backend/migrations/000024_ai_usage_prompt_version.down.sql b/backend/migrations/000024_ai_usage_prompt_version.down.sql new file mode 100644 index 0000000..73e4b33 --- /dev/null +++ b/backend/migrations/000024_ai_usage_prompt_version.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS ai_usage_prompt_version_idx; +ALTER TABLE ai_usage DROP COLUMN IF EXISTS prompt_version; diff --git a/backend/migrations/000024_ai_usage_prompt_version.up.sql b/backend/migrations/000024_ai_usage_prompt_version.up.sql new file mode 100644 index 0000000..77a92ae --- /dev/null +++ b/backend/migrations/000024_ai_usage_prompt_version.up.sql @@ -0,0 +1,4 @@ +ALTER TABLE ai_usage ADD COLUMN prompt_version TEXT NULL; +CREATE INDEX ai_usage_prompt_version_idx + ON ai_usage (prompt_version, created_at DESC) + WHERE prompt_version IS NOT NULL;