//! Per-model token accounting. Entry point for the ModelMatrix work //! the aibridge `context::estimate_tokens` deprecation has been pointing //! at. Starts minimal — just `estimate_tokens` — so call sites can //! migrate off the deprecated helper. Extend with per-model context //! windows, max_tokens defaults, provider hints, etc. as we move the //! rest of `aibridge::context::known_windows` over. /// Namespace for per-model token + context accounting. Methods are /// associated functions — no instance required — because the underlying /// estimates are deterministic and stateless. pub struct ModelMatrix; impl ModelMatrix { /// Rough token count — char count divided by 4, rounded up. This /// is the same heuristic OpenAI's cookbook uses for English text; /// it's within ±15% of BPE tokenizers for code + prose and doesn't /// require a tokenizer lookup. Good enough for budget math where /// the goal is "don't blow the context window" rather than exact /// billing. /// /// Moved from `aibridge::context::estimate_tokens` (still there with /// a `#[deprecated]` pointer — callers should migrate here). Empty /// string → 0; one char → 1 (ceiling of 1/4 = 1). pub fn estimate_tokens(text: &str) -> usize { (text.chars().count() + 3) / 4 } } #[cfg(test)] mod tests { use super::ModelMatrix; #[test] fn empty_string_is_zero_tokens() { assert_eq!(ModelMatrix::estimate_tokens(""), 0); } #[test] fn three_chars_is_one_token() { // 3 → ceil(3/4) = 1. Matches the deprecated helper's behavior // so the migration is a drop-in replacement. assert_eq!(ModelMatrix::estimate_tokens("abc"), 1); } #[test] fn four_chars_is_one_token() { assert_eq!(ModelMatrix::estimate_tokens("abcd"), 1); } #[test] fn five_chars_is_two_tokens() { assert_eq!(ModelMatrix::estimate_tokens("abcde"), 2); } #[test] fn counts_chars_not_bytes() { // Multi-byte UTF-8 chars count as 1 char each — important for // prompts with emoji or non-ASCII text. "héllo" is 5 chars // (5 unicode scalars) → ceil(5/4) = 2 tokens, same as "hello". assert_eq!(ModelMatrix::estimate_tokens("héllo"), 2); assert_eq!(ModelMatrix::estimate_tokens("📚📚📚📚"), 1); // 4 chars } #[test] fn large_text_scales_linearly() { assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(400)), 100); assert_eq!(ModelMatrix::estimate_tokens(&"x".repeat(401)), 101); } }