From eab5f264079cce493d30bae4b4a368e29e6a54d4 Mon Sep 17 00:00:00 2001 From: vikingowl Date: Sun, 5 Apr 2026 21:53:05 +0200 Subject: [PATCH] test: tokenizer cached encoding path coverage --- internal/tokenizer/tokenizer_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go index 2db6226..1665cb4 100644 --- a/internal/tokenizer/tokenizer_test.go +++ b/internal/tokenizer/tokenizer_test.go @@ -45,3 +45,15 @@ func TestTokenizer_CodeCountsReasonably(t *testing.T) { t.Errorf("code token count out of expected range: %d", n) } } + +func TestTokenizer_CachedLoadReturnsSameResult(t *testing.T) { + tok := tokenizer.New("cl100k_base") + first := tok.Count("hello world") + second := tok.Count("hello world") + if first != second { + t.Errorf("cached result differs: first=%d second=%d", first, second) + } + if first <= 0 { + t.Errorf("expected positive token count, got %d", first) + } +}