vikingowl
58beb7ce3c
Phase 4 routing decisions depend on knowing whether the SLM classifier is actually firing or whether the heuristic is silently doing all the work. Adds the instrumentation to make that observable. router.ClassifierSource enum (heuristic / slm / slm_fallback) is set on Task by every classifier: - HeuristicClassifier → ClassifierHeuristic - slm.Classifier → ClassifierSLM on success, ClassifierSLMFallback when the SLM call fails or returns unparseable output The source is plumbed through router.Outcome to QualityTracker, which now maintains per-source counters alongside the existing per-arm × task EMA scores. QualitySnapshot serializes both (classifier_counts is omitempty for back-compat with pre-feature quality.json files). lazyClassifier logs at INFO the first time it falls back to heuristic because the SLM hasn't booted yet — distinguishes operational fallback from an unconfigured-SLM run. slm.Manager.Start() now records elapsed-to-healthy and the main.go goroutine logs it as part of the "SLM ready" event. Confirms whether short-lived runs are racing the boot cycle. New `gnoma router stats` subcommand prints both tables (arm × task quality, classifier source breakdown) from quality.json with a Phase 4 trust hint when the data is too sparse or the SLM share is low. 6 new tests cover ClassifierSource string/enum, heuristic + SLM source propagation, QualityTracker counter round-trip, and back-compat restore from a legacy quality.json without classifier_counts.
97 lines
2.9 KiB
Go
97 lines
2.9 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"somegit.dev/Owlibou/gnoma/internal/message"
|
|
)
|
|
|
|
// TestHeuristicClassifier_ParityWithClassifyTask verifies that
|
|
// HeuristicClassifier.Classify produces identical results to ClassifyTask.
|
|
func TestHeuristicClassifier_ParityWithClassifyTask(t *testing.T) {
|
|
prompts := []string{
|
|
"debug the failing test",
|
|
"explain how generics work",
|
|
"implement a new HTTP handler",
|
|
"refactor the auth middleware",
|
|
"security audit the login flow",
|
|
"write unit tests for the parser",
|
|
"scaffold a new service",
|
|
"plan the migration strategy",
|
|
"orchestrate the deployment pipeline",
|
|
"review the pull request",
|
|
}
|
|
|
|
cls := HeuristicClassifier{}
|
|
ctx := context.Background()
|
|
var noHistory []message.Message
|
|
|
|
for _, p := range prompts {
|
|
want := ClassifyTask(p)
|
|
got, err := cls.Classify(ctx, p, noHistory)
|
|
if err != nil {
|
|
t.Errorf("Classify(%q) unexpected error: %v", p, err)
|
|
continue
|
|
}
|
|
if got.Type != want.Type {
|
|
t.Errorf("Classify(%q).Type = %s, want %s", p, got.Type, want.Type)
|
|
}
|
|
if got.ComplexityScore != want.ComplexityScore {
|
|
t.Errorf("Classify(%q).ComplexityScore = %v, want %v", p, got.ComplexityScore, want.ComplexityScore)
|
|
}
|
|
if got.RequiresTools != want.RequiresTools {
|
|
t.Errorf("Classify(%q).RequiresTools = %v, want %v", p, got.RequiresTools, want.RequiresTools)
|
|
}
|
|
if got.Priority != want.Priority {
|
|
t.Errorf("Classify(%q).Priority = %v, want %v", p, got.Priority, want.Priority)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHeuristicClassifier_SetsClassifierSource(t *testing.T) {
|
|
got, err := HeuristicClassifier{}.Classify(context.Background(), "implement foo", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.ClassifierSource != ClassifierHeuristic {
|
|
t.Errorf("ClassifierSource = %v, want ClassifierHeuristic", got.ClassifierSource)
|
|
}
|
|
}
|
|
|
|
func TestClassifierSource_String(t *testing.T) {
|
|
cases := []struct {
|
|
src ClassifierSource
|
|
want string
|
|
}{
|
|
{ClassifierUnknown, "unknown"},
|
|
{ClassifierHeuristic, "heuristic"},
|
|
{ClassifierSLM, "slm"},
|
|
{ClassifierSLMFallback, "slm_fallback"},
|
|
{ClassifierSource(999), "unknown"},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := tc.src.String(); got != tc.want {
|
|
t.Errorf("ClassifierSource(%d).String() = %q, want %q", tc.src, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestHeuristicClassifier_IgnoresHistory verifies that history has no effect
|
|
// on the heuristic classifier (it operates only on the prompt).
|
|
func TestHeuristicClassifier_IgnoresHistory(t *testing.T) {
|
|
cls := HeuristicClassifier{}
|
|
ctx := context.Background()
|
|
prompt := "implement a binary search function"
|
|
|
|
withoutHistory, _ := cls.Classify(ctx, prompt, nil)
|
|
withHistory, _ := cls.Classify(ctx, prompt, []message.Message{
|
|
{Role: message.RoleUser, Content: []message.Content{{Type: message.ContentText, Text: "previous message"}}},
|
|
})
|
|
|
|
if withoutHistory.Type != withHistory.Type {
|
|
t.Errorf("history should not affect HeuristicClassifier: got %s vs %s",
|
|
withoutHistory.Type, withHistory.Type)
|
|
}
|
|
}
|