diff --git a/.env.example b/.env.example index 3671f56d3..0959be327 100644 --- a/.env.example +++ b/.env.example @@ -89,3 +89,9 @@ REDDIT_CONVERSIONS_TOKEN= # CAPROVER_APP=monoscope # CAPROVER_APP_TOKEN= # CAPROVER_PASSWORD= + +# --- LLM observability ------------------------------------------------------ +# The GenAI conventions' opt-in for recording prompt and completion text on the assistant's +# own chat spans: NO_CONTENT (default) | SPAN_ONLY | EVENT_ONLY | SPAN_AND_EVENT. Prompts carry +# customers' telemetry, so leave it off wherever the server exports its telemetry to a shared project. +# OTEL_INSTRUMENTATION_GENAI_CAPTURE_CONTENT=SPAN_ONLY diff --git a/docs/llm-observability.md b/docs/llm-observability.md new file mode 100644 index 000000000..3c3321cdd --- /dev/null +++ b/docs/llm-observability.md @@ -0,0 +1,156 @@ +# LLM & agent observability + +Monoscope's AI view: conversations, agent runs, tool calls, tokens and cost, from +any OpenTelemetry producer, beside the HTTP/DB spans of the same trace. + +## Principles + +1. **OTel first, every dialect accepted.** We store the GenAI semantic conventions + (`gen_ai.*`, semconv-genai 1.42-dev). Every other dialect in the wild (OpenInference, + OpenLLMetry/Traceloop, Vercel AI SDK `ai.*`, Langfuse, Logfire, Claude Code) is + normalised onto those keys **once, at ingest**, so every surface — explorer, KQL, + dashboards, monitors, alerts, the AI assistant — reads one vocabulary. Raw keys are + kept (additive, as `migrateHttpSemanticConventions` already does for HTTP/DB). +2. **Cost is an attribute, not a report.** `gen_ai.usage.cost` (USD) is stamped on every + priced span at ingest, with `gen_ai.usage.cost_source` = `reported` | `estimated`. + A producer-reported cost wins. That makes `summarize sum(attributes.gen_ai.usage.cost)` + work in every existing surface with no new query path. +3. **Reuse the product.** Traces, the span drawer, Table, dashboards and the AI dock are + extended, not forked. The AI page is a list + detail over the same spans. +4. **Signals before LLM judges.** Deterministic, free detectors (tool errors, truncation, + refusals, loops, context pressure, retry storms) run on every conversation; LLM + analysis is one click away through the existing assistant. + +## Ingest normalisation (`Pkg.GenAI`, pure, doctested) + +Hooked into `OtlpServer.keyValueToJSON`, so spans, span events and log records (Claude +Code exports `claude_code.api_request` as a *log*) all pass through it. A key-prefix +guard keeps the non-AI hot path to one scan. + +| Canonical | Accepted sources (first wins) | +|---|---| +| `gen_ai.operation.name` | `openinference.span.kind`, `traceloop.span.kind` + `llm.request.type`, Vercel `ai.operationId` | +| `gen_ai.provider.name` | `gen_ai.system`, `llm.provider`, `llm.system`, `ai.model.provider` | +| `gen_ai.request.model` / `response.model` | `llm.model_name`, `llm.request.model_name`, `ai.model.id`, `ai.response.model`, Claude Code `model` | +| `gen_ai.usage.input_tokens` | `gen_ai.usage.prompt_tokens`, `llm.token_count.prompt`, `ai.usage.promptTokens`/`inputTokens`, `input_tokens` | +| `gen_ai.usage.output_tokens` | `gen_ai.usage.completion_tokens`, `llm.token_count.completion`, `ai.usage.completionTokens`/`outputTokens`, `output_tokens` | +| `gen_ai.usage.cache_read.input_tokens` | `gen_ai.usage.cache_read_input_tokens`, `llm.token_count.prompt_details.cache_read`, `cache_read_tokens` | +| `gen_ai.usage.cache_write.input_tokens` | `gen_ai.usage.cache_creation.input_tokens`, `gen_ai.usage.cache_creation_input_tokens`, `llm.token_count.prompt_details.cache_write`, `cache_creation_tokens` | +| `gen_ai.usage.reasoning.output_tokens` | `gen_ai.usage.reasoning_tokens`, `llm.token_count.completion_details.reasoning` | +| `gen_ai.usage.cost` (+ `cost_source`) | `llm.cost.total`, `gen_ai.cost.total_tokens`, `cost_usd`; else estimated | +| `gen_ai.input.messages` / `output.messages` | JSON strings parsed to arrays; OpenInference `llm.{input,output}_messages.N.message.*`, OpenLLMetry `gen_ai.{prompt,completion}.N.*`, Vercel `ai.prompt.messages`/`ai.response.text`, `input.value`/`output.value` | +| `gen_ai.conversation.id` | `session.id`, `langfuse.session.id`, `traceloop.association.properties.session_id`, `ai.telemetry.metadata.sessionId` | +| `gen_ai.agent.name` | `agent.name`, `traceloop.workflow.name`, `ai.telemetry.functionId` | +| `gen_ai.tool.name` / `call.id` / `call.arguments` / `call.result` | `tool.name`, `ai.toolCall.{name,id,args,result}`, `tool_name`, `tool_use_id` | +| `gen_ai.response.finish_reasons` | `llm.finish_reason`, `ai.response.finishReason`, `gen_ai.completion.0.finish_reason`, `stop_reason` | + +Indexed message keys are folded into one messages array and dropped (otherwise the +dot-nesting stores them as `{"0":{…}}` objects nobody can query). + +### Pricing + +`static/model_prices.json`: a trimmed, generated copy of LiteLLM's MIT price table +(`scripts/update-model-prices.sh`), embedded at compile time. Lookup normalises the model id +(lowercase, no provider path, Bedrock region/vendor prefix or `-v1:0` revision, no `@` pin), +then tries the id without a date suffix, the longest known family prefix, and the nearest dated +release of a bare alias. Cost: + +``` +cached = cache_read + cache_write +uncached = input − cached if input ≥ cached (GenAI conventions: input includes the cache) + = input otherwise (Anthropic-native: input excludes it) +cost = (uncached·in + cache_read·read + cache_write·write + output·out) / 1e6, ≥ 0 +``` + +Claude Code's counts are exclusive by definition and are made inclusive at ingest. Only +inference operations (`chat`, `text_completion`, `generate_content`, `embeddings`) are priced: +agent spans may carry aggregate usage, and pricing them would count every call twice. +Dashboards that sum *tokens* must filter on those operations for the same reason. + +Unknown models carry tokens but no cost; the UI says "unpriced" rather than `$0`. + +## Surfaces + +### Everywhere spans already render +- **Summary pills** (`generateSpanSummary`): model, `1.2k→340 tok`, `$0.0042`, tool name, + finish reason when abnormal. The log explorer, live tail and waterfall pick these up. +- **Span drawer → "Conversation" tab** for any gen_ai span: messages as a transcript + (system collapsed, tool calls with arguments, tool results), usage/cost header. +- **Trace header:** total LLM cost and tokens when the trace has gen_ai spans. + +### LLM & Agents page (`/p/:pid/llm`, `/llm/conversations/:key`, `/llm/dashboard`) +- **KPI strip:** conversations, model calls, tokens, cost (with the count of unpriced calls), + error rate, p95 model latency; cost and calls charted by model. +- **Two lists.** *Conversations* are groups with a `gen_ai.conversation.id` or an + `invoke_agent` span; *LLM calls* lists every inference span. Single-call traces (a + classifier, a judge) appear only in the second, so they don't drown real conversations. +- **Conversations table** (shared `Table`): title (first user message), agent, models, + turns, tool calls, tokens, cost, errors/signals, duration, last seen. Grouping key is + `gen_ai.conversation.id`, falling back to the trace id; a trace adopts the id any of its + spans carries (frameworks put it only on the root agent span). Facets: agent, model, + signal. +- **Conversation detail:** + - *Transcript*: each new message once (LLM inputs repeat the history; we diff), with + per-step model, latency, tokens, cost chips, tool calls inline with their results. + - *Timeline*: AI spans in execution order with duration bars and a cumulative-cost + line; each row opens the existing span drawer / trace. + - *Signals*: deterministic findings with the step that caused them. + - *Ask AI*: opens the assistant dock seeded with this conversation. +- **Dashboard** (`llm-observability.yaml`, read-only template): cost by model, tokens by + type, calls and p95 latency by model, error rate, top tools and tool errors, cost by agent, + most expensive conversations. +- **Setup** (zero state): one-paste snippets for OTel GenAI (Python/Node/Go/Java/.NET), + OpenLLMetry, OpenInference, Vercel AI SDK, Pydantic AI/Logfire, LangChain/LangGraph, + OpenAI Agents SDK, CrewAI, Mastra, Google ADK, Strands, Agno, Bedrock/Azure/Vertex, + Claude Code CLI, Claude Console push, MCP, plain OTLP/HTTP. + +### Signals (deterministic) +| Signal | Rule | +|---|---| +| Tool error | `execute_tool` span with error status | +| Truncated | finish reason `length` / `max_tokens` | +| Refusal / filtered | finish reason `content_filter` / `refusal`, Claude Code `api_refusal` | +| Loop | same tool + arguments ≥ 3 times in a conversation | +| Context pressure | input tokens ≥ 80 % of the model's context window | +| LLM error | inference span with error status | + +## Dogfooding + +Monoscope's own assistant is an agent: the copilot dock, issue analysis, Slack +investigations, routines and the log-pattern judges all call `Data.Effectful.LLM`. +- `runLLMReal` calls the OpenAI client directly (langchain-hs discards `usage`) and wraps each + call in a `chat {model}` span with the canonical attributes and real token usage. +- The agent loop opens `invoke_agent monoscope-assistant` (with `gen_ai.conversation.id`) + and an `execute_tool {name}` span per tool call. +- Our self-telemetry project therefore shows every assistant conversation, its cost and its + tools — the first customer of the feature. + +### Content capture +Our own spans record prompts and completions only when +`OTEL_INSTRUMENTATION_GENAI_CAPTURE_CONTENT` is `SPAN_ONLY`/`SPAN_AND_EVENT` (the GenAI +conventions' switch). Production leaves it off — our prompts carry customers' telemetry — so +the self-telemetry project shows tokens, cost, models and tools there, and transcripts in dev. + +## Tests +- Doctests: dialect normalisation per dialect, model-id normalisation, cost formula, + provided-cost precedence, transcript diffing. +- Integration (`AIObservabilitySpec`): ingest one conversation mixing dialects through the + real OTLP path; assert normalised attributes and cost on the stored rows, the conversations + list row, transcript, signals, and golden HTML. +- Dogfood: run the assistant against golden LLM responses with an in-memory span exporter, + replay its spans through ingest, and assert the AI page shows the conversation. + +## Follow-ups (not in this change) +- Promote `gen_ai.usage.*`, model and conversation id to TimeFusion columns. +- NL-defined signals compiled to classifiers (Raindrop-style) on the issues pipeline. +- Topic clustering of first user messages via Drain. +- Resolve `gen_ai.*_ref` content URIs. +- Bill `projects.ai_usage` from the provider's real usage: return it from the `LLM` effect + (spans already carry it) and delete `estimatedTokens`. +- Aggregate list totals in SQL (GROUP BY the conversation key) instead of over the newest + 5,000 spans; the pages say when they hit that cap. +- Normalise per dialect in a pre-pass (detector + alias table + fixups) so new dialects do not + touch the shared derivation; stamp an `inference` marker so KQL filters stop listing operations. +- Interpret `LLM` above `Tracing` so inference spans use the `Tracing` effect, and share one + TLS client across calls. +- Signals defined in natural language (Raindrop-style), topic clusters of first messages. diff --git a/hpack-includes/lib-deps.yaml b/hpack-includes/lib-deps.yaml index 0f375b5e6..30515b1bf 100644 --- a/hpack-includes/lib-deps.yaml +++ b/hpack-includes/lib-deps.yaml @@ -150,6 +150,8 @@ - yaml - indexed-traversable - hs-opentelemetry-sdk +- hs-opentelemetry-exporter-in-memory +- hs-opentelemetry-exporter-otlp - hs-opentelemetry-api - hs-opentelemetry-instrumentation-wai - hs-opentelemetry-instrumentation-auto diff --git a/monoscope.cabal b/monoscope.cabal index d90846546..f9c7e38bd 100644 --- a/monoscope.cabal +++ b/monoscope.cabal @@ -328,6 +328,7 @@ extra-source-files: static/public/dashboards/http-stats.yaml static/public/dashboards/kitchensink.yaml static/public/dashboards/kubernetes.yaml + static/public/dashboards/llm-observability.yaml static/public/dashboards/minio.yaml static/public/dashboards/mysql.yaml static/public/dashboards/nginx.yaml @@ -338,6 +339,7 @@ extra-source-files: static/public/dashboards/redis.yaml static/public/dashboards/rum.yaml static/public/dashboards/timefusion.yaml + static/model_prices.json src/Data/Effectful/CodeReview/proof.sh source-repository head @@ -428,9 +430,11 @@ library Pages.Containers Pages.Dashboards Pages.Endpoints + Pages.GenAIMessages Pages.GitSync Pages.Infrastructure Pages.Issues + Pages.LLM Pages.LogExplorer.LiveTail Pages.LogExplorer.Log Pages.LogExplorer.LogItem @@ -457,6 +461,7 @@ library Pkg.EmailTemplates Pkg.ErrorFingerprint Pkg.ExtractionWorker + Pkg.GenAI Pkg.Git Pkg.ImpactReview Pkg.IngestBudget @@ -622,6 +627,8 @@ library , hasql-th , hasql-transaction , hs-opentelemetry-api + , hs-opentelemetry-exporter-in-memory + , hs-opentelemetry-exporter-otlp , hs-opentelemetry-instrumentation-auto , hs-opentelemetry-instrumentation-hasql , hs-opentelemetry-instrumentation-wai @@ -831,6 +838,7 @@ test-suite integration-tests EvalConformanceSpec MinioManagerSpec MonitoringSpec + Opentelemetry.GenAISpec Opentelemetry.IngestionSpec Pages.BotsSpec Pages.CodeContextSpec @@ -1042,9 +1050,11 @@ test-suite test-dev Pages.Containers Pages.Dashboards Pages.Endpoints + Pages.GenAIMessages Pages.GitSync Pages.Infrastructure Pages.Issues + Pages.LLM Pages.LogExplorer.LiveTail Pages.LogExplorer.Log Pages.LogExplorer.LogItem @@ -1071,6 +1081,7 @@ test-suite test-dev Pkg.EmailTemplates Pkg.ErrorFingerprint Pkg.ExtractionWorker + Pkg.GenAI Pkg.Git Pkg.ImpactReview Pkg.IngestBudget @@ -1112,6 +1123,7 @@ test-suite test-dev EvalConformanceSpec MinioManagerSpec MonitoringSpec + Opentelemetry.GenAISpec Opentelemetry.IngestionSpec Pages.BotsSpec Pages.CodeContextSpec @@ -1265,6 +1277,8 @@ test-suite test-dev , hasql-th , hasql-transaction , hs-opentelemetry-api + , hs-opentelemetry-exporter-in-memory + , hs-opentelemetry-exporter-otlp , hs-opentelemetry-instrumentation-auto , hs-opentelemetry-instrumentation-hasql , hs-opentelemetry-instrumentation-wai diff --git a/package.yaml b/package.yaml index 2f4b7a7d9..44c09a8cb 100644 --- a/package.yaml +++ b/package.yaml @@ -41,6 +41,7 @@ extra-source-files: - static/public/assets/web-components/dist/manifest.json # Dashboard templates are spliced by TH the same way. - static/public/dashboards/*.yaml + - static/model_prices.json # The frozen proof check is spliced into Data.Effectful.CodeReview.Explanation. - src/Data/Effectful/CodeReview/proof.sh # - static/public/**/*.png diff --git a/scripts/update-model-prices.py b/scripts/update-model-prices.py new file mode 100755 index 000000000..ce0cc5706 --- /dev/null +++ b/scripts/update-model-prices.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Regenerate static/model_prices.json from LiteLLM's MIT-licensed price table. + +Output: {"model": [input, output, cache_read, cache_write, context_window]}, prices in +USD per million tokens, keys normalised the way Pkg.GenAI.normaliseModel looks them up. +""" +import json, re, sys, urllib.request + +SRC = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +# First provider wins when two keys normalise to the same model id. +PREFERRED = ["openai", "anthropic", "gemini", "vertex_ai-language-models", "mistral", "deepseek", "xai", "groq", "cohere_chat", "bedrock_converse", "bedrock", "azure"] +MODES = {"chat", "completion", "responses", "embedding"} + + +def normalise(key): + k = key.lower().rsplit("/", 1)[-1] + k = re.sub(r"^(?:[a-z0-9-]+\.)+(?=(?:claude|llama|mistral|nova|titan|command|jamba))", "", k) + return re.sub(r"[@:].*$", "", re.sub(r"-v\d+:.*$", "", k)) + + +def per_million(v): + return None if v is None else round(v * 1_000_000, 6) + + +def main(): + data = json.load(urllib.request.urlopen(SRC)) + rank = lambda kv: PREFERRED.index(kv[1].get("litellm_provider")) if kv[1].get("litellm_provider") in PREFERRED else len(PREFERRED) + out = {} + for key, v in sorted(((k, v) for k, v in data.items() if isinstance(v, dict)), key=rank): + if v.get("mode") not in MODES or v.get("input_cost_per_token") is None: + continue + out.setdefault(normalise(key), [ + per_million(v["input_cost_per_token"]), + per_million(v.get("output_cost_per_token", 0)), + per_million(v.get("cache_read_input_token_cost")), + per_million(v.get("cache_creation_input_token_cost")), + v.get("max_input_tokens"), + ]) + path = sys.argv[1] if len(sys.argv) > 1 else "static/model_prices.json" + with open(path, "w") as f: + json.dump(dict(sorted(out.items())), f, separators=(",", ":")) + f.write("\n") + print(f"{len(out)} models -> {path}") + + +main() diff --git a/src/Data/Effectful/LLM.hs b/src/Data/Effectful/LLM.hs index 3fdb785ad..5facd743b 100644 --- a/src/Data/Effectful/LLM.hs +++ b/src/Data/Effectful/LLM.hs @@ -6,8 +6,10 @@ module Data.Effectful.LLM ( openAIEmbeddings, embedDocumentsBounded, embedDocumentsWithProgress, - callOpenAIAPI, modelAndEffort, + LLMTelemetry (..), + GenAIContentCapture (..), + capturesContent, runLLMReal, runLLMGolden, stripCodeBlock, @@ -19,7 +21,9 @@ import Control.Monad.Except (throwError) import Data.Aeson qualified as AE import Data.Aeson.Lens (key) import Data.ByteString qualified as BS +import Data.Default (Default (..)) import Data.Effectful.Wreq (withGoldenCache) +import Data.HashMap.Strict qualified as HM import Data.IntMap.Strict qualified as IntMap import Data.Text qualified as T import Data.Text.Foreign qualified as TextForeign @@ -32,11 +36,17 @@ import Langchain.DocumentLoader.Core qualified as DocLoader import Langchain.Embeddings.Core qualified as EmbCore import Langchain.Embeddings.OpenAI qualified as EmbOAI import Langchain.LLM.Core qualified as LLMCore -import Langchain.LLM.OpenAI qualified as OpenAI +import Langchain.LLM.OpenAICompatible () +import OpenAI.V1 (Methods (..), getClientEnv, makeMethods) import OpenAI.V1.Chat.Completions qualified as OpenAIV1 import OpenAI.V1.Models qualified as Models +import OpenAI.V1.Usage qualified as Usage +import OpenTelemetry.Attributes qualified as OA +import OpenTelemetry.Trace.Core qualified as Trace import Relude import Relude.Extra.Bifunctor (firstF) +import System.Envy (Var (..)) +import UnliftIO.Exception (tryAny) -- Generic-derived JSON for LLMCore.Message (record field names match the prior @@ -180,25 +190,106 @@ embedDocumentsWithProgress request save docs = runExceptT $ concat . reverse <$> data EmbeddingMean = EmbeddingMean !Int !Int !(VU.Vector Double) +-- | @OTEL_INSTRUMENTATION_GENAI_CAPTURE_CONTENT@, the GenAI conventions' opt-in for recording +-- prompt and completion text. Off by default: our prompts carry customers' telemetry. +data GenAIContentCapture = NoContent | SpanOnly | EventOnly | SpanAndEvent + deriving stock (Bounded, Enum, Eq, Generic, Show) + + +instance Default GenAIContentCapture where + def = NoContent + + +instance Var GenAIContentCapture where + fromVar v = find ((== v) . toVar) universe + toVar = \case + NoContent -> "NO_CONTENT" + SpanOnly -> "SPAN_ONLY" + EventOnly -> "EVENT_ONLY" + SpanAndEvent -> "SPAN_AND_EVENT" + + +data LLMTelemetry = LLMTelemetry {tracerProvider :: Trace.TracerProvider, capture :: GenAIContentCapture} + + -- | Real interpreter that makes actual OpenAI API calls -runLLMReal :: IOE :> es => Eff (LLM ': es) a -> Eff es a -runLLMReal = interpret $ \_ -> \case - CallLLM model prompt apiKey -> liftIO $ callOpenAIAPI model prompt apiKey - CallAgenticChat history params apiKey -> liftIO do - let openAI = OpenAI.OpenAI{apiKey, callbacks = [], baseUrl = Nothing} - result <- LLMCore.chat openAI history (Just params) - pure $ first show result +runLLMReal :: IOE :> es => LLMTelemetry -> Eff (LLM ': es) a -> Eff es a +runLLMReal t = interpret $ \_ -> \case + CallLLM model prompt apiKey -> liftIO $ fmap replyText <$> completeChat t apiKey (promptParams model prompt) + CallAgenticChat history params apiKey -> liftIO $ fmap LLMCore.from <$> completeChat t apiKey (withHistory history params) EmbedDocuments config docs -> liftIO $ embedDocumentsBounded (firstF show . EmbCore.embedDocuments config) docs -- | Golden test interpreter that caches responses -runLLMGolden :: IOE :> es => FilePath -> Eff (LLM ': es) a -> Eff es a -runLLMGolden goldenDir = interpret $ \_ -> \case - CallLLM model prompt apiKey -> liftIO $ getOrCreateGoldenResponse goldenDir model prompt apiKey - CallAgenticChat history params apiKey -> liftIO $ getOrCreateAgenticGoldenResponse goldenDir history params apiKey +runLLMGolden :: IOE :> es => LLMTelemetry -> FilePath -> Eff (LLM ': es) a -> Eff es a +runLLMGolden t goldenDir = interpret $ \_ -> \case + CallLLM model prompt apiKey -> liftIO $ inference t (promptParams model prompt) (\r -> contentAttr t "gen_ai.output.messages" ([AE.object ["role" AE..= ("assistant" :: Text), "content" AE..= r]] :: [AE.Value])) $ getOrCreateGoldenResponse goldenDir model prompt apiKey + CallAgenticChat history params apiKey -> liftIO $ inference t (withHistory history params) (contentAttr t "gen_ai.output.messages" . pure @[] . LLMCore.to @(OpenAIV1.Message (V.Vector OpenAIV1.Content))) $ getOrCreateAgenticGoldenResponse goldenDir history params apiKey EmbedDocuments config docs -> liftIO $ getOrCreateEmbeddingGoldenResponse goldenDir config docs +withHistory :: LLMCore.ChatHistory -> OpenAIV1.CreateChatCompletion -> OpenAIV1.CreateChatCompletion +withHistory history params = params{OpenAIV1.messages = V.fromList $ map LLMCore.to $ toList history} + + +-- | A GenAI inference span (semconv-genai) around one chat completion. It writes to the +-- provider directly: the background stack interprets 'Tracing' before this effect. +inference :: LLMTelemetry -> OpenAIV1.CreateChatCompletion -> (a -> [(Text, OA.Attribute)]) -> IO (Either Text a) -> IO (Either Text a) +inference t params describe call = + Trace.inSpan' (Trace.makeTracer t.tracerProvider "monoscope.llm" Trace.tracerOptions) ("chat " <> model) Trace.defaultSpanArguments{Trace.kind = Trace.Client, Trace.attributes = HM.fromList requestAttrs} \sp -> do + result <- call + case result of + Left err -> Trace.setStatus sp (Trace.Error err) + Right reply -> Trace.addAttributes sp $ HM.fromList $ describe reply + pure result + where + Models.Model model = params.model + requestAttrs = + [("gen_ai.operation.name", "chat"), ("gen_ai.provider.name", "openai"), ("gen_ai.request.model", OA.toAttribute model)] + <> [("gen_ai.request.reasoning.level", OA.toAttribute $ T.toLower $ T.drop 16 $ show e) | Just e <- [params.reasoning_effort]] + <> contentAttr t "gen_ai.input.messages" params.messages + + +capturesContent :: GenAIContentCapture -> Bool +capturesContent = (`elem` [SpanOnly, SpanAndEvent]) + + +-- | A message attribute, recorded only under content capture. +contentAttr :: AE.ToJSON a => LLMTelemetry -> Text -> a -> [(Text, OA.Attribute)] +contentAttr t k v = [(k, OA.toAttribute $ decodeUtf8 @Text $ AE.encode v) | capturesContent t.capture] + + +-- | langchain-hs drops the provider's usage, so call the client directly to keep it. +completeChat :: LLMTelemetry -> Text -> OpenAIV1.CreateChatCompletion -> IO (Either Text (OpenAIV1.Message Text)) +completeChat t apiKey params = (>>= firstMessage) <$> inference t params usageAttrs (requestChat apiKey params) + where + usageAttrs OpenAIV1.ChatCompletionObject{OpenAIV1.id = responseId, OpenAIV1.model = Models.Model responseModel, OpenAIV1.usage = Usage.Usage{Usage.prompt_tokens, Usage.completion_tokens, Usage.prompt_tokens_details, Usage.completion_tokens_details}, OpenAIV1.choices} = + [ ("gen_ai.response.id", OA.toAttribute responseId) + , ("gen_ai.response.model", OA.toAttribute responseModel) + , ("gen_ai.usage.input_tokens", tokens prompt_tokens) + , ("gen_ai.usage.output_tokens", tokens completion_tokens) + , ("gen_ai.response.finish_reasons", OA.toAttribute $ V.toList $ V.map (\OpenAIV1.Choice{OpenAIV1.finish_reason} -> finish_reason) choices) + ] + <> [("gen_ai.usage.cache_read.input_tokens", tokens n) | Just Usage.PromptTokensDetails{Usage.cached_tokens = Just n} <- [prompt_tokens_details]] + <> [("gen_ai.usage.reasoning.output_tokens", tokens n) | Just Usage.CompletionTokensDetails{Usage.reasoning_tokens = Just n} <- [completion_tokens_details]] + <> contentAttr t "gen_ai.output.messages" (V.map (\OpenAIV1.Choice{OpenAIV1.message} -> message) choices) + tokens = OA.toAttribute @Int . fromIntegral + + +requestChat :: Text -> OpenAIV1.CreateChatCompletion -> IO (Either Text OpenAIV1.ChatCompletionObject) +requestChat apiKey params = do + Methods{createChatCompletion} <- (\env -> makeMethods env apiKey Nothing Nothing) <$> getClientEnv "https://api.openai.com" + first (("LLM Error: " <>) . show) <$> tryAny (createChatCompletion params) + + +firstMessage :: OpenAIV1.ChatCompletionObject -> Either Text (OpenAIV1.Message Text) +firstMessage OpenAIV1.ChatCompletionObject{OpenAIV1.choices} = maybe (Left "LLM Error: no choices returned") (\OpenAIV1.Choice{OpenAIV1.message} -> Right message) (V.headM choices) + + +replyText :: OpenAIV1.Message Text -> Text +replyText = LLMCore.content . LLMCore.from + + -- | Sanitize text for use in filenames (replaces special chars with underscores) sanitizeForFilename :: Text -> String sanitizeForFilename = toString . T.replace "/" "_" . T.replace "|" "_" . T.replace " " "_" . T.replace "\n" "_" . T.take 50 @@ -222,7 +313,7 @@ data LLMResponse = LLMResponse getOrCreateGoldenResponse :: FilePath -> Text -> Text -> Text -> IO (Either Text Text) getOrCreateGoldenResponse goldenDir model prompt apiKey = withGoldenCache goldenDir (promptToFilename prompt) ("Failed to decode LLM response from file: " <>) (.llmResponse) $ do - response <- callOpenAIAPI model prompt apiKey + response <- fmap replyText . (>>= firstMessage) <$> requestChat apiKey (promptParams model prompt) pure (LLMResponse{llmPrompt = prompt, llmResponse = response}, response) @@ -251,32 +342,15 @@ modelAndEffort spec = case T.breakOnEnd "#" spec of _ -> Nothing --- | Actual OpenAI API call implementation -callOpenAIAPI :: Text -> Text -> Text -> IO (Either Text Text) -callOpenAIAPI model fullPrompt apiKey = do - let openAI = - OpenAI.OpenAI - { OpenAI.apiKey = apiKey - , OpenAI.callbacks = [] - , OpenAI.baseUrl = Nothing - } - userMessage = - OpenAIV1.User - { OpenAIV1.content = V.fromList [OpenAIV1.Text{OpenAIV1.text = fullPrompt}] - , OpenAIV1.name = Nothing - } - (modelName, effort) = modelAndEffort model - params = - OpenAIV1._CreateChatCompletion - { OpenAIV1.model = modelName - , OpenAIV1.messages = V.fromList [userMessage] - , OpenAIV1.reasoning_effort = effort - } - -- Use langchain-hs to generate response - result <- liftIO $ LLMCore.generate openAI fullPrompt (Just params) - case result of - Left err -> pure $ Left $ "LLM Error: " <> show err - Right response -> pure $ Right response +promptParams :: Text -> Text -> OpenAIV1.CreateChatCompletion +promptParams model prompt = + OpenAIV1._CreateChatCompletion + { OpenAIV1.model = modelName + , OpenAIV1.messages = V.fromList [OpenAIV1.User{OpenAIV1.content = V.fromList [OpenAIV1.Text{OpenAIV1.text = prompt}], OpenAIV1.name = Nothing}] + , OpenAIV1.reasoning_effort = effort + } + where + (modelName, effort) = modelAndEffort model -- | Create deterministic cache key from user query and iteration @@ -306,9 +380,8 @@ hasToolsInParams p = isJust $ AE.toJSON p ^? key "tools" getOrCreateAgenticGoldenResponse :: FilePath -> LLMCore.ChatHistory -> OpenAIV1.CreateChatCompletion -> Text -> IO (Either Text LLMCore.Message) getOrCreateAgenticGoldenResponse goldenDir history params apiKey = withGoldenCache goldenDir (cacheKeyFromQuery historyMessages hasTools <> ".json") ("Failed to decode agentic chat cache from: " <>) (.accResponse) $ do - let openAI = OpenAI.OpenAI{apiKey, callbacks = [], baseUrl = Nothing} - Models.Model modelName = params.model - response <- first show <$> LLMCore.chat openAI history (Just params) + let Models.Model modelName = params.model + response <- fmap LLMCore.from . (>>= firstMessage) <$> requestChat apiKey (withHistory history params) pure (AgenticChatCache{accHistory = historyMessages, accModel = modelName, accHasTools = hasTools, accResponse = response}, response) where historyMessages = toList history diff --git a/src/Models/Apis/LogQueries.hs b/src/Models/Apis/LogQueries.hs index 6070a4124..3e35824b5 100644 --- a/src/Models/Apis/LogQueries.hs +++ b/src/Models/Apis/LogQueries.hs @@ -1,4 +1,5 @@ module Models.Apis.LogQueries ( + percentile, errHashPrefix, patHashPrefix, cursorEpsilon, diff --git a/src/Models/Telemetry/Telemetry.hs b/src/Models/Telemetry/Telemetry.hs index 3c81194b1..8e8e7a008 100644 --- a/src/Models/Telemetry/Telemetry.hs +++ b/src/Models/Telemetry/Telemetry.hs @@ -100,6 +100,7 @@ module Models.Telemetry.Telemetry ( SpanEvent (..), atMapText, atMapInt, + getNestedValue, spanServiceName, spanAttr, mkSystemLog, @@ -176,6 +177,7 @@ import Models.Projects.Projects qualified as Projects import Pkg.DeriveUtils (AesonText (..), DB, UUIDId (..), WrappedEnum (..), WrappedEnumSC (..), idFromText, unAesonTextMaybe) import Pkg.ErrorFingerprint qualified as EF import Pkg.ExtractionWorker qualified as EW +import Pkg.GenAI qualified as GenAI import Pkg.Metrics qualified as Metrics import Relude hiding (ask) import Relude.Extra.Bifunctor (bimapF) @@ -183,7 +185,7 @@ import Relude.Extra.Foldable1 (maximum1, minimum1) import System.Logging qualified as Log import System.Tracing (forkWithCtx) import Text.Casing (fromAny, toKebab) -import Utils (classifyUserAgent, encodeText, extractMessageFromLog, formatBytes, getDurationNSMS, jsonToMap, lookupValueText, nonEmptyT, scrubNulText, scrubNulValue) +import Utils (classifyUserAgent, encodeText, extractMessageFromLog, formatBytes, getDurationNSMS, jsonToMap, lookupValueText, nonEmptyT, scrubNulText, scrubNulValue, tokenFlow) import Web.HttpApiData (FromHttpApiData (..), ToHttpApiData (..)) import "cryptonite" Crypto.Hash qualified as Crypto @@ -2856,6 +2858,14 @@ generateSpanSummary otel = httpStatus = atMapInt "http.response.status_code" attrsM httpMethod = atMapText "http.request.method" attrsM hasHttp = isJust httpMethod || isJust httpStatus + genAIOp = atMapText "gen_ai.operation.name" attrsM + genAINumber k = + attrsM >>= getNestedValue (T.split (== '.') k) >>= \case + AE.Number n -> Just (Sci.toRealFloat n :: Double) + _ -> Nothing + tokens = (,) <$> atMapInt "gen_ai.usage.input_tokens" attrsM <*> atMapInt "gen_ai.usage.output_tokens" attrsM + abnormalFinish = + attrsM >>= getNestedValue ["gen_ai", "response", "finish_reasons"] >>= AET.parseMaybe AE.parseJSON >>= \(GenAI.FinishReasons rs) -> find (isJust . GenAI.abnormalFinish) rs durMs d = toText (getDurationNSMS (fromIntegral d)) statusCodeStyle code @@ -2916,6 +2926,8 @@ generateSpanSummary otel = <$> requestType <|> tag "kind" "neutral" "database" <$ guard (isJust dbSys) + <|> tag "kind" "neutral" "ai" + <$ genAIOp <|> tag "kind" "neutral" "internal" <$ guard (otel.kind == Just "internal") @@ -2943,12 +2955,15 @@ generateSpanSummary otel = , tag "rpc.service" "neutral" <$> atMapText "rpc.service" attrsM , spanNameFallback , errorStatus "badge-error" + , tag "gen_ai.response.finish_reasons" "badge-warning" <$> abnormalFinish , tag "attributes" "text-textWeak" . encTrunc 500 <$> mfilter (not . Map.null) attrsM , errorStatus "right-badge-error" , dbBadge <$> dbSys , (atMapInt "http.response.body.size" attrsM <|> atMapInt "http.response_content_length" attrsM) >>= \n -> guard (n > 0) $> tag "size" "right-badge-neutral" (formatBytes n) , tag "protocol" "right-badge-neutral" "http" <$ guard hasHttp , tag "protocol" "right-badge-neutral" "rpc" <$ rpcMethod + , tag "gen_ai.usage" "right-badge-neutral" <$> (tokens >>= \(i, o) -> tokenFlow (Just $ fromIntegral i) (Just $ fromIntegral o)) + , tag "gen_ai.usage.cost" "right-badge-neutral" . GenAI.formatUsd <$> genAINumber "gen_ai.usage.cost" , tag "duration" "right-badge-neutral" . durMs <$> otel.duration ] <> identitySummary attrsM diff --git a/src/Opentelemetry/OtlpServer.hs b/src/Opentelemetry/OtlpServer.hs index 86f7d83ea..2611c40c6 100644 --- a/src/Opentelemetry/OtlpServer.hs +++ b/src/Opentelemetry/OtlpServer.hs @@ -80,6 +80,7 @@ import Network.GRPC.Server.StreamType (Methods (..), fromMethods) import OpenTelemetry.Attributes qualified as OA import OpenTelemetry.Trace (TracerProvider) import Pkg.DeriveUtils (AesonText (..), unAesonTextMaybe, unUUIDId) +import Pkg.GenAI qualified as GenAI import Pkg.LiveTail qualified as LiveTail import Pkg.Metrics (wireTypeErrorsRef) import Pkg.Metrics qualified as Metrics @@ -1004,6 +1005,12 @@ keyValueToJSON :: [PC.KeyValue] -> AE.Value keyValueToJSON !kvs = attrPairsToJSON $ migrateHttpSemanticConventions $ keyValuePairs kvs +-- | A span's or log record's own attributes, where LLM telemetry lives; metric, scope, event +-- and link attributes keep their producer's keys so metric series identity never shifts. +recordAttributesToJSON :: [PC.KeyValue] -> AE.Value +recordAttributesToJSON = attrPairsToJSON . GenAI.normalizeGenAI . migrateHttpSemanticConventions . keyValuePairs + + anyValueToJSON :: PC.AnyValue -> AE.Value anyValueToJSON av = case av ^. PCF.maybe'value of @@ -1119,7 +1126,7 @@ convertLogRecordToOtelLog !fallbackTime !pid resourceM logRecord = , severity_number = severityNumber } , body = Just $ AesonText $ anyValueToJSON $ logRecord ^. PLF.body - , attributes = fmap AesonText $ jsonToMap $ removeProjectId $ keyValueToJSON $ logRecord ^. PLF.attributes + , attributes = fmap AesonText $ jsonToMap $ removeProjectId $ recordAttributesToJSON $ logRecord ^. PLF.attributes , resource = fmap AesonText $ jsonToMap $ removeProjectId $ resourceToJSON resourceM , hashes = Just V.empty , kind = Just "log" @@ -1215,7 +1222,7 @@ convertSpanToOtelLog !fallbackTime !pid resourceM pSpan = ] | link <- lks ] - !attributes = jsonToMap $ removeProjectId $ keyValueToJSON $ pSpan ^. PTF.attributes + !attributes = jsonToMap $ removeProjectId $ recordAttributesToJSON $ pSpan ^. PTF.attributes spanName' = pSpan ^. PTF.name -- "monoscope.http" included so re-ingested spans get consistent body/attribute processing isOurSdkSpan = spanName' `elem` Telemetry.sdkSpanNames diff --git a/src/Pages/BodyWrapper.hs b/src/Pages/BodyWrapper.hs index 1af6d316f..2334c1c43 100644 --- a/src/Pages/BodyWrapper.hs +++ b/src/Pages/BodyWrapper.hs @@ -124,6 +124,7 @@ menu lang pid = , (I18n.t lang "nav.monitors", (urls.monitors pid).listGet Nothing Nothing, "list-check") , ("Real User Monitoring", urls.rumRedirect pid [] [], "web") , (I18n.t lang "nav.infrastructure", (urls.traces pid).hostsGetH Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing, "server") + , ("LLM & Agents", (urls.traces pid).llmGetH Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing, "sparkles") , (I18n.t lang "nav.api_catalog", apiCatalogUrl pid Nothing, "swap") , ("Repositories", urls.repositoriesGet pid Nothing Nothing, "code-branch") ] @@ -1317,6 +1318,7 @@ navFlyoutItems pid = \case "Explorer" -> [(label, link pid) | (label, link) <- Components.explorerTabs] "Infrastructure" -> [(label, link pid Nothing Nothing Nothing) | (label, link) <- Components.infrastructureTabs] "Real User Monitoring" -> [("Dashboard", urls.rumRedirect pid [] []), ("Sessions", urls.rumRedirect pid ["sessions"] [])] + "LLM & Agents" -> [(label, (urls.traces pid).llmGetH (Just view) Nothing Nothing Nothing Nothing Nothing Nothing Nothing) | (label, view) <- [("Conversations", "conversations"), ("LLM calls", "calls")]] <> [("Dashboard", urls.llmDashboardRedirect pid [])] -- API Docs carries no host: it is the whole project's learned spec. The -- per-host and per-endpoint scopes are reached from the catalog rows. "API Catalog" -> [("Incoming", apiCatalogUrl pid (Just "Incoming")), ("Outgoing", apiCatalogUrl pid (Just "Outgoing")), ("API Docs", urls.apiCatalogDocs pid Nothing (Just "Incoming") Nothing)] diff --git a/src/Pages/GenAIMessages.hs b/src/Pages/GenAIMessages.hs new file mode 100644 index 000000000..339e53e2b --- /dev/null +++ b/src/Pages/GenAIMessages.hs @@ -0,0 +1,162 @@ +-- | GenAI messages rendered from the conventions' parts shape: the span drawer's +-- Conversation tab and the conversation transcript share these. +module Pages.GenAIMessages ( + Call (..), + genAIAt, + genAINum, + message_, + isGenAISpan, + genAIUsage, + genAISpanDetail_, +) where + +import Data.Aeson qualified as AE +import Data.Aeson.Encode.Pretty qualified as AEP +import Data.Aeson.KeyMap qualified as KM +import Data.Map.Strict qualified as M +import Data.Scientific (toRealFloat) +import Data.Text qualified as T +import Data.Time (UTCTime) +import Data.UUID qualified as UUID +import Hasql.Interpolate qualified as HI +import Lucid +import Models.Ids qualified as Projects +import Models.Telemetry.Telemetry qualified as Telemetry +import Pages.Components (BadgeColor (..), metaChip_, tonePill_) +import Pkg.DeriveUtils (AesonText) +import Pkg.GenAI qualified as GenAI +import Pkg.Parser.Eval (jsonAsText) +import Relude +import Utils (drawerRowAttrs_, faSprite_, getDurationNSMS, nonEmptyT, prettyPrintCount, tokenFlow) +import Web.Links + + +-- | One GenAI span without its message bodies: the list views read thousands of these. +data Call = Call + { id :: Text + , traceId :: Text + , timestamp :: UTCTime + , duration :: Int64 + , failed :: Bool + , name :: Text + , service :: Maybe Text + , operation :: Text + , conversation :: Maybe Text + , agent :: Maybe Text + , model :: Maybe Text + , inputTokens :: Maybe Double + , outputTokens :: Maybe Double + , cost :: Maybe Double + , tool :: Maybe Text + , toolArgs :: Maybe Text + , preview :: Maybe Text + , finish :: Maybe (AesonText GenAI.FinishReasons) + , user :: Maybe Text + , attributes :: Maybe (AesonText (Map Text AE.Value)) + -- ^ Read only for a conversation's detail: it carries the message bodies. + } + deriving stock (Generic, Show) + deriving anyclass (HI.DecodeRow) + + +genAIAt :: Maybe (Map Text AE.Value) -> [Text] -> Maybe AE.Value +genAIAt attrs ks = attrs >>= Telemetry.getNestedValue ("gen_ai" : ks) + + +genAINum :: Maybe (Map Text AE.Value) -> [Text] -> Maybe Double +genAINum attrs ks = genAIAt attrs ks >>= \case AE.Number n -> Just (toRealFloat n); AE.String t -> readMaybe (toString t); _ -> Nothing + + +-- | Tool arguments and results usually arrive as JSON strings; indent them when they parse. +prettyJson :: AE.Value -> Text +prettyJson v = maybe (jsonAsText v) (decodeUtf8 . AEP.encodePretty) $ case v of + AE.String t -> AE.decodeStrict @AE.Value (encodeUtf8 t) + _ -> Just v + + +message_ :: Projects.ProjectId -> Map Text Call -> Maybe Call -> AE.Value -> Html () +message_ pid tools stepM m = case role of + "system" -> details_ [class_ "rounded-lg border border-strokeWeak bg-fillWeaker px-3 py-2 text-sm"] do + summary_ [class_ "cursor-pointer text-xs font-medium text-textWeak"] "System instructions" + div_ [class_ "mt-2 max-h-96 overflow-auto"] parts_ + _ -> article_ [class_ $ "flex flex-col gap-1.5 rounded-lg border px-3 py-2 " <> if role == "user" then "border-strokeBrand-weak bg-fillBrand-weak/40" else "border-strokeWeak bg-bgRaised"] do + div_ [class_ "flex flex-wrap items-center gap-2 text-xs"] do + span_ [class_ "font-semibold uppercase tracking-wide text-textWeak"] $ toHtml $ T.toTitle role + whenJust stepM \c -> do + span_ [class_ "text-textWeak"] $ toHtml $ T.intercalate " · " $ catMaybes [c.model, Just $ toText $ getDurationNSMS $ fromIntegral c.duration, tokenFlow c.inputTokens c.outputTokens, GenAI.formatUsd <$> c.cost] + when c.failed $ tonePill_ ErrorBadge "Error" + whenJust (UUID.fromText c.id) \u -> button_ ([type_ "button", class_ "ms-auto text-textBrand hover:underline"] <> drawerRowAttrs_ ((urls.logExplorer pid).logExplorerItemDetailedGet u c.timestamp Nothing (Just "tab-genai") Nothing True)) "Open span" + parts_ + where + role = case m of AE.Object o | Just (AE.String r) <- KM.lookup "role" o -> r; _ -> "message" + parts = case m of AE.Object o | Just (AE.Array ps) <- KM.lookup "parts" o -> toList ps; _ -> [m] + parts_ = div_ [class_ "flex flex-col gap-2"] $ forM_ parts (part_ tools) + + +part_ :: Map Text Call -> AE.Value -> Html () +part_ tools p = case (field "type", p) of + (Just "text", _) -> textBlock_ $ maybe "" jsonAsText $ fieldV "content" + (Just "reasoning", _) -> details_ [class_ "text-sm"] $ summary_ [class_ "cursor-pointer text-xs text-textWeak"] "Reasoning" >> textBlock_ (maybe "" jsonAsText $ fieldV "content") + (Just "tool_call", _) -> div_ [class_ "flex flex-col gap-1 rounded-md border border-strokeWeak bg-fillWeaker px-2 py-1.5 font-mono text-xs"] do + div_ [class_ "flex flex-wrap items-center gap-2"] do + faSprite_ "function" "regular" "h-3 w-3 text-iconNeutral" + span_ [class_ "font-semibold text-textStrong"] $ toHtml $ fromMaybe "tool" $ field "name" + whenJust (field "id" >>= (`M.lookup` tools)) \t -> do + span_ [class_ "text-textWeak"] $ toHtml $ getDurationNSMS $ fromIntegral t.duration + when t.failed $ tonePill_ ErrorBadge "Failed" + whenJust (fieldV "arguments") $ pre_ [class_ "max-h-48 overflow-auto whitespace-pre-wrap break-all text-textWeak"] . toHtml . prettyJson + (Just "tool_call_response", _) -> details_ [class_ "rounded-md border border-strokeWeak px-2 py-1.5 text-xs"] do + summary_ [class_ "cursor-pointer font-mono text-textWeak"] $ toHtml $ "Result" <> maybe "" ((" · " <>) . (\t -> fromMaybe t.name t.tool)) (field "id" >>= (`M.lookup` tools)) <> " · " <> show (T.length response) <> " chars" + pre_ [class_ "mt-1 max-h-96 overflow-auto whitespace-pre-wrap break-all font-mono text-textWeak"] $ toHtml $ maybe "" prettyJson $ fieldV "response" + (_, AE.String t) -> textBlock_ t + _ -> pre_ [class_ "overflow-auto whitespace-pre-wrap break-all font-mono text-xs text-textWeak"] $ toHtml $ jsonAsText p + where + fieldV k = case p of AE.Object o -> KM.lookup (fromString $ toString k) o; _ -> Nothing + field k = fieldV k >>= \case AE.String t -> Just t; _ -> Nothing + response = maybe "" jsonAsText $ fieldV "response" + textBlock_ = div_ [class_ "whitespace-pre-wrap break-words text-sm text-textStrong"] . toHtml + + +-- * Span drawer + + +-- | A model call's tokens and cost in one line, for the waterfall and other dense rows. +genAIUsage :: Maybe (Map Text AE.Value) -> Maybe Text +genAIUsage attrs = nonEmptyT $ Just $ T.intercalate " · " $ catMaybes [tokenFlow (number ["usage", "input_tokens"]) (number ["usage", "output_tokens"]), GenAI.formatUsd <$> number ["usage", "cost"]] + where + number = genAINum attrs + + +isGenAISpan :: Maybe (Map Text AE.Value) -> Bool +isGenAISpan attrs = isJust $ genAIAt attrs ["operation", "name"] + + +-- | The drawer's Conversation tab for one GenAI span: usage, then its messages or tool I/O. +genAISpanDetail_ :: Projects.ProjectId -> Maybe (Map Text AE.Value) -> Html () +genAISpanDetail_ pid attrs = div_ [class_ "flex flex-col gap-3 p-2"] do + div_ [class_ "flex flex-wrap gap-1.5"] $ forM_ facts $ uncurry metaChip_ + whenJust (at ["system_instructions"]) \v -> message_ pid M.empty Nothing (AE.object ["role" AE..= ("system" :: Text), "parts" AE..= ([AE.object ["type" AE..= ("text" :: Text), "content" AE..= jsonAsText v]] :: [AE.Value])]) + forM_ (list "input" <> list "output") (message_ pid M.empty Nothing) + when (op == Just "execute_tool") do + whenJust (at ["tool", "call", "arguments"]) \v -> part_ M.empty (AE.object ["type" AE..= ("tool_call" :: Text), "name" AE..= txt ["tool", "name"], "arguments" AE..= v]) + whenJust (at ["tool", "call", "result"]) \v -> part_ M.empty (AE.object ["type" AE..= ("tool_call_response" :: Text), "response" AE..= v]) + where + at = genAIAt attrs + txt ks = at ks >>= \case AE.String t -> Just t; _ -> Nothing + op = txt ["operation", "name"] + list dir = case at [dir, "messages"] of Just (AE.Array ms) -> toList ms; _ -> [] + number = genAINum attrs + facts = + catMaybes + [ ("Operation",) <$> op + , ("Provider",) <$> txt ["provider", "name"] + , ("Model",) <$> (txt ["response", "model"] <|> txt ["request", "model"]) + , ("Agent",) <$> txt ["agent", "name"] + , ("Tool",) <$> txt ["tool", "name"] + , ("Input tokens",) . prettyPrintCount . round <$> number ["usage", "input_tokens"] + , ("Output tokens",) . prettyPrintCount . round <$> number ["usage", "output_tokens"] + , ("Cache read",) . prettyPrintCount . round <$> number ["usage", "cache_read", "input_tokens"] + , (\c -> ("Cost", GenAI.formatUsd c <> maybe "" (" " <>) (txt ["usage", "cost_source"]))) <$> number ["usage", "cost"] + , ("Finish",) . jsonAsText <$> at ["response", "finish_reasons"] + , ("Conversation",) <$> txt ["conversation", "id"] + ] diff --git a/src/Pages/LLM.hs b/src/Pages/LLM.hs new file mode 100644 index 000000000..757f9bc88 --- /dev/null +++ b/src/Pages/LLM.hs @@ -0,0 +1,507 @@ +{-# LANGUAGE OrPatterns #-} + +-- | LLM & agent observability: conversations, model calls, tools, tokens and cost, read +-- from GenAI spans that 'Pkg.GenAI' normalised at ingest (docs/llm-observability.md). +module Pages.LLM ( + llmGetH, + conversationGetH, + LLMGet (..), + LLMBody (..), + ConversationGet (..), + Conversation (..), + Signal (..), + groupConversations, +) where + +import Data.Aeson qualified as AE +import Data.Effectful.Hasql qualified as Hasql +import Data.Foldable1 qualified as F1 +import Data.HashSet qualified as HS +import Data.Map.Strict qualified as M +import Data.Text qualified as T +import Data.Time (UTCTime, addUTCTime, diffUTCTime) +import Data.UUID qualified as UUID +import Data.Vector qualified as V +import Hasql.Interpolate qualified as HI +import Lucid +import Lucid.Aria qualified as Aria +import Lucid.Htmx (hxPost_, hxSwap_, hxTarget_, hxVals_) +import Lucid.Hyperscript (__) +import Models.Apis.LogQueries qualified as LogQueries +import Models.Ids qualified as Projects +import Pages.BodyWrapper (BWConfig (..), PageCtx (..), mkPageCtx, navTabAttrs) +import Pages.Components (BadgeColor (..), Deferred (..), EmptyStateAction (..), RefreshingDeferred (..), callout_, localTimeFmt_, metaChip_, tableSkeleton_, tonePill_, withDeferredBody) +import Pages.Containers (mkWindow, windowed) +import Pages.GenAIMessages (Call (..), genAIAt, message_) +import Pkg.Components.Table (Column, Features (..), Table (..), ZeroState (..), col, countedFilter, facetActions, inventoryTable, withAttrs) +import Pkg.Components.TimePicker qualified as TimePicker +import Pkg.Components.Widget qualified as Widget +import Pkg.DeriveUtils (AesonText (..), unAesonTextMaybe) +import Pkg.GenAI qualified as GenAI +import Relude +import System.Types (ATAuthCtx, RespHeaders, addRespHeaders, useTfReads) +import Utils (TabStrip (..), drawerRowAttrs_, faSprite_, getDurationNSMS, navTabStrip_, prettyPrintCount, tokenFlow) +import Web.Links + + +data Signal = ToolError | ModelError | Truncated | Filtered | Looping | ContextPressure + deriving stock (Bounded, Enum, Eq, Ord, Show) + + +signalLabel :: Signal -> Text +signalLabel = \case + ToolError -> "Tool error" + ModelError -> "Model error" + Truncated -> "Truncated" + Filtered -> "Filtered" + Looping -> "Repeated tool call" + ContextPressure -> "Context ≥80%" + + +signalHelp :: Signal -> Text +signalHelp = \case + ToolError -> "A tool call failed." + ModelError -> "A model call returned an error." + Truncated -> "A reply stopped at the output token limit." + Filtered -> "A reply was refused or cut by a content filter." + Looping -> "The same tool ran with the same arguments three or more times." + ContextPressure -> "A call used at least 80% of the model's context window." + + +data Conversation = Conversation + { key :: Text + , title :: Maybe Text + , started :: UTCTime + , lastSeen :: UTCTime + , agents :: [Text] + , models :: [Text] + , services :: [Text] + , users :: [Text] + , turns :: Int + , llmCalls :: Int + , toolCalls :: Int + , tokens :: Double + , cost :: Maybe Double + , errors :: Int + , activeNs :: Int64 + , signals :: [Signal] + } + deriving stock (Generic, Show) + + +inference :: Call -> Bool +inference = GenAI.isInferenceOp . (.operation) + + +abnormal :: Call -> [GenAI.Abnormal] +abnormal c = maybe [] (\(AesonText (GenAI.FinishReasons rs)) -> mapMaybe GenAI.abnormalFinish rs) c.finish + + +-- | A trace joins the conversation any of its spans names; frameworks often set the id on the +-- root agent span only. A trace with no id is its own conversation if it ran an agent, and +-- otherwise lists only as calls, so single classifier calls don't drown real conversations. +-- +-- >>> import Relude.Unsafe qualified as Unsafe +-- >>> let t = Unsafe.read @UTCTime "2026-01-01 00:00:00 UTC" +-- >>> let c tr op conv = Call "s" tr t 1000 False op Nothing op conv Nothing (Just "gpt-4o") (Just 10) (Just 5) (Just 0.01) Nothing Nothing (Just "hi") Nothing Nothing Nothing +-- >>> map (\x -> (x.key, x.turns, x.llmCalls)) $ groupConversations $ V.fromList [c "t1" "invoke_agent" (Just "conv"), c "t1" "chat" Nothing, c "t2" "chat" (Just "conv"), c "t3" "chat" Nothing, c "t4" "invoke_agent" Nothing] +-- [("conv",2,2),("t4",1,0)] +groupConversations :: V.Vector Call -> [Conversation] +groupConversations calls = sortOn (Down . (.lastSeen)) $ mapMaybe conversation $ M.toList grouped + where + -- Logs carry no trace (Claude Code's events): each is its own unit, never one shared "" trace. + traces = M.fromListWith (<>) [(if T.null c.traceId then c.id else c.traceId, [c]) | c <- V.toList calls] + traceKey (unit, spans) = asum (map (.conversation) spans) <|> (unit <$ guard (any ((`elem` ["invoke_agent", "invoke_workflow"]) . (.operation)) spans)) + grouped = M.fromListWith (<>) [(k, [spans]) | (unit, spans) <- M.toList traces, Just k <- [traceKey (unit, spans)]] + conversation (k, traceGroups) = do + let spans = sortOn (.timestamp) $ concat traceGroups + chats = filter inference spans + tools = filter ((== "execute_tool") . (.operation)) spans + distinct f = ordNub $ mapMaybe f spans + end c = addUTCTime (fromIntegral c.duration / 1e9) c.timestamp + traceSpan ss = fromMaybe 0 $ do + ne <- nonEmpty ss + pure $ round $ 1e9 * diffUTCTime (F1.maximum $ fmap end ne) (F1.minimum $ fmap (.timestamp) ne) + first' <- viaNonEmpty head spans + pure + Conversation + { key = k + , title = asum $ map (.preview) spans + , started = first'.timestamp + , lastSeen = maybe first'.timestamp (F1.maximum . fmap end) (nonEmpty spans) + , agents = distinct (.agent) + , models = ordNub $ mapMaybe (.model) chats + , services = distinct (.service) + , users = distinct (.user) + , turns = length traceGroups + , llmCalls = length chats + , toolCalls = length tools + , tokens = sum $ concatMap (\c -> catMaybes [c.inputTokens, c.outputTokens]) chats + , cost = viaNonEmpty sum (mapMaybe (.cost) chats) + , errors = length $ filter (.failed) spans + , activeNs = sum $ map traceSpan traceGroups + , -- Deterministic rules, in 'Signal' order, so each finding is explainable and free. + signals = + [ sig + | (sig, True) <- + zip + universe + [ any (\c -> c.failed && c.operation == "execute_tool") spans + , any (\c -> c.failed && inference c) spans + , any ((GenAI.Truncated `elem`) . abnormal) spans + , any ((GenAI.Filtered `elem`) . abnormal) spans + , any (>= 3) $ M.fromListWith (+) [((t, c.toolArgs), 1 :: Int) | c <- spans, Just t <- [c.tool]] + , any pressured chats + ] + ] + } + pressured c = fromMaybe False do + window <- c.model >>= GenAI.lookupPrice >>= (.contextWindow) + used <- c.inputTokens + pure $ used >= 0.8 * fromIntegral window + + +-- | The window's GenAI spans, newest first and capped so a busy project stays one bounded scan. +-- Lists skip the attributes blob, where message bodies live; a conversation's detail reads it. +selectCalls :: Projects.ProjectId -> TimePicker.TimeWindow -> Bool -> HI.Sql -> ATAuthCtx [Call] +selectCalls pid window bodies scope = do + tf <- useTfReads + Hasql.withHasqlTimefusion tf + $ Hasql.interp + $ [HI.sql| + SELECT id::text, COALESCE(context___trace_id, ''), timestamp, COALESCE(duration, 0), COALESCE(status_code = 'ERROR', false), COALESCE(name, ''), resource___service___name, + attributes -> 'gen_ai' -> 'operation' ->> 'name', + attributes -> 'gen_ai' -> 'conversation' ->> 'id', + attributes -> 'gen_ai' -> 'agent' ->> 'name', + COALESCE(attributes -> 'gen_ai' -> 'response' ->> 'model', attributes -> 'gen_ai' -> 'request' ->> 'model'), + (attributes -> 'gen_ai' -> 'usage' ->> 'input_tokens')::float8, + (attributes -> 'gen_ai' -> 'usage' ->> 'output_tokens')::float8, + (attributes -> 'gen_ai' -> 'usage' ->> 'cost')::float8, + attributes -> 'gen_ai' -> 'tool' ->> 'name', + left(attributes -> 'gen_ai' -> 'tool' -> 'call' ->> 'arguments', 300), + attributes -> 'gen_ai' -> 'input' ->> 'preview', + attributes -> 'gen_ai' -> 'response' -> 'finish_reasons', + attributes___user___id, |] + <> bool [HI.sql|CASE WHEN false THEN attributes END|] [HI.sql|attributes|] bodies + <> [HI.sql| + FROM otel_logs_and_spans + WHERE project_id = #{pid.toText} AND timestamp BETWEEN #{window.fromTime} AND #{window.toTime} + AND attributes -> 'gen_ai' -> 'operation' ->> 'name' IS NOT NULL |] + <> scope + <> [HI.sql| ORDER BY timestamp DESC LIMIT 5000|] + + +data View = ConversationsView | CallsView + deriving stock (Bounded, Enum, Eq) + + +viewParam :: View -> Text +viewParam = \case + ConversationsView -> "conversations" + CallsView -> "calls" + + +data LLMBody = ConversationsBody (Table Conversation) | CallsBody (Table Call) + + +instance ToHtml LLMBody where + toHtml = \case + ConversationsBody t -> toHtml t + CallsBody t -> toHtml t + toHtmlRaw = toHtml + + +newtype LLMGet = LLMPage (PageCtx (Deferred LLMBody)) + deriving (ToHtml) via (PageCtx (RefreshingDeferred LLMBody)) + + +llmGetH :: Projects.ProjectId -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Text -> ATAuthCtx (RespHeaders LLMGet) +llmGetH pid viewM agentM modelM signalM fromP toP sinceP deferredM = do + (_, _, bw) <- mkPageCtx pid + window <- mkWindow fromP toP sinceP + let view = fromMaybe ConversationsView $ find ((== viewM) . Just . viewParam) universe + link v a m s = windowed window ((urls.traces pid).llmGetH (Just $ viewParam v) a m s) + url = link view agentM modelM signalM (Just "1") + want m xs = maybe True (`elem` xs) m + body <- withDeferredBody deferredM "llmContainer" url (tableSkeleton_ 8) do + calls <- V.fromList <$> selectCalls pid window False [HI.sql||] + let conversations = V.fromList $ groupConversations calls + capped = bool "" " (newest 5,000 spans)" (V.length calls >= 5000) + kpis = kpis_ (V.length conversations) calls + reset = link view Nothing Nothing Nothing Nothing + zero = ZeroState{icon = "sparkles", title = "No LLM or agent telemetry yet", description = "Send OpenTelemetry GenAI spans from any SDK or framework — OpenLLMetry, OpenInference, Vercel AI SDK, Pydantic AI, LangChain, the OpenAI Agents SDK, Claude Code — and conversations, cost and tool calls appear here.", action = ESLink "https://monoscopehq.com/docs/sdks/llm-observability/" "Instrument your agent"} + table, base :: Table a + table = inventoryTable "llm" url + base = table{features = table.features{header = Just kpis, zeroState = Just zero, searchPlaceholder = Just $ case view of ConversationsView -> "Search conversations"; CallsView -> "Search model calls"}} + pure $ case view of + ConversationsView -> + let all' = conversations + shown = V.filter (\c -> want agentM c.agents && want modelM c.models && want signalM (map signalLabel c.signals)) all' + t = base + in ConversationsBody + t + { columns = conversationColumns + , rows = shown + , features = + t.features + { rowLink = Just \c -> windowed window ((urls.traces pid).llmConversationGetH c.key) + , resultSummary = Just $ show (V.length shown) <> " of " <> show (V.length all') <> " conversations" <> capped + , facets = + Just + $ facetActions + reset + "llmContainer" + [ countedFilter "Agent" "agent" agentM (.agents) all' + , countedFilter "Model" "model" modelM (.models) all' + , countedFilter "Signal" "signal" signalM (map signalLabel . (.signals)) all' + ] + } + } + CallsView -> + -- Frameworks name the agent on its root span only; each call takes its trace's. + let agents = M.fromList [(c.traceId, a) | c <- V.toList calls, Just a <- [c.agent]] + all' = V.map (\c -> c{agent = c.agent <|> M.lookup c.traceId agents}) $ V.filter inference calls + shown = V.filter (\c -> want agentM (maybeToList c.agent) && want modelM (maybeToList c.model)) all' + t = base + in CallsBody + t + { columns = callColumns + , rows = shown + , features = + t.features + { rowAttrs = Just \c -> maybe [] (\u -> drawerRowAttrs_ $ (urls.logExplorer pid).logExplorerItemDetailedGet u c.timestamp Nothing (Just "tab-genai") Nothing True) (UUID.fromText c.id) + , resultSummary = Just $ show (V.length shown) <> " of " <> show (V.length all') <> " model calls" <> capped + , facets = Just $ facetActions reset "llmContainer" [countedFilter "Agent" "agent" agentM (maybeToList . (.agent)) all', countedFilter "Model" "model" modelM (maybeToList . (.model)) all'] + } + } + addRespHeaders + $ LLMPage + $ PageCtx + bw + { pageTitle = "LLM & Agents" + , menuItem = Just "LLM & Agents" + , navTabs = Just $ navTabStrip_ NavLinks "LLM views" "" [(toHtml (viewLabel v), link v Nothing Nothing Nothing Nothing, v == view) | v <- universe] + , pageActions = Just $ div_ [class_ "inline-flex items-center gap-2", data_ "default-window" "24H"] do + a_ [class_ "btn btn-sm", href_ $ urls.llmDashboardRedirect pid []] $ faSprite_ "dashboard" "regular" "h-3.5 w-3.5" >> "Dashboard" + TimePicker.liveDataControls_ Nothing window.currentRange Nothing TimePicker.RefreshOnly + , needsGridStack = True + } + body + where + viewLabel = \case + ConversationsView -> "Conversations" :: Text + CallsView -> "LLM calls" + + -- \| Headline numbers over model calls only: agent spans carry aggregate usage that would count twice. + kpis_ :: Int -> V.Vector Call -> Html () + kpis_ conversationCount calls = div_ [class_ "flex flex-col gap-3 px-4 pt-3"] do + div_ [class_ "grid grid-cols-6 gap-px overflow-hidden rounded-lg border border-strokeWeak bg-strokeWeak max-xl:grid-cols-3 max-sm:grid-cols-2"] do + stat "Conversations" (show conversationCount) Nothing + stat "Model calls" (prettyPrintCount n) Nothing + stat "Tokens" (prettyPrintCount $ round $ sum $ concatMap (\c -> catMaybes [c.inputTokens, c.outputTokens]) chats) Nothing + stat "Cost" (maybe "—" (GenAI.formatUsd . sum) $ nonEmpty $ mapMaybe (.cost) chats) $ (\k -> show k <> " unpriced") <$> mfilter (> 0) (Just $ length $ filter (isNothing . (.cost)) chats) + stat "Error rate" (if n == 0 then "—" else show (round (100 * fromIntegral (length $ filter (.failed) chats) / fromIntegral n :: Double) :: Int) <> "%") Nothing + stat "p95 latency" (if n == 0 then "—" else toText $ getDurationNSMS $ fromIntegral $ LogQueries.percentile 0.95 $ V.fromList $ Relude.sort $ map (.duration) chats) Nothing + div_ [class_ "grid grid-cols-2 gap-3 max-lg:grid-cols-1"] $ forM_ charts $ div_ [class_ "min-h-56"] . Widget.widget_ + where + chats = filter inference $ V.toList calls + n = length chats + stat :: Text -> Text -> Maybe Text -> Html () + stat label value note = div_ [class_ "flex flex-col gap-0.5 bg-bgRaised px-3 py-2"] do + span_ [class_ "text-xs text-textWeak"] $ toHtml label + div_ [class_ "flex items-baseline gap-1.5"] do + span_ [class_ "text-lg font-semibold tabular-nums text-textStrong"] $ toHtml value + whenJust note $ span_ [class_ "text-xs text-textWeak", term "data-tippy-content" "Calls to models without a published price carry tokens but no cost."] . toHtml + inferenceKql = "attributes.gen_ai.operation.name in (\"chat\", \"text_completion\", \"generate_content\", \"embeddings\")" + charts = + [ Widget.infraTimeseries pid "llm-cost" "Cost by model" "$" $ inferenceKql <> " | summarize sum(attributes.gen_ai.usage.cost) by bin_auto(timestamp), attributes.gen_ai.request.model" + , Widget.infraTimeseries pid "llm-calls" "Model calls by model" "" $ inferenceKql <> " | summarize count(*) by bin_auto(timestamp), attributes.gen_ai.request.model" + ] + + conversationColumns :: [Column Conversation] + conversationColumns = + [ col "Conversation" titleCell & withAttrs [class_ "min-w-72"] + , col "Agent" (chips . (.agents)) & withAttrs [class_ "w-36 max-lg:hidden"] + , col "Models" (chips . (.models)) & withAttrs [class_ "w-40 max-xl:hidden"] + , col "Turns" (num . fromIntegral . (.turns)) & withAttrs [class_ "w-16 text-right"] + , col "Calls" (\c -> span_ [class_ "whitespace-nowrap tabular-nums text-textWeak"] $ toHtml $ show c.llmCalls <> " model · " <> show c.toolCalls <> " tool") & withAttrs [class_ "w-32 text-right max-md:hidden"] + , col "Tokens" (num . (.tokens)) & withAttrs [class_ "w-20 text-right"] + , col "Cost" (\c -> span_ [class_ "tabular-nums text-textStrong"] $ toHtml $ maybe "—" GenAI.formatUsd c.cost) & withAttrs [class_ "w-20 text-right"] + , col "Signals" signalsCell & withAttrs [class_ "w-44"] + , col "Last seen" (localTimeFmt_ "MMM dd, HH:mm" . (.lastSeen)) & withAttrs [class_ "w-28 max-md:hidden"] + ] + where + titleCell, signalsCell :: Conversation -> Html () + titleCell c = div_ [class_ "flex min-w-0 flex-col"] do + span_ [class_ "truncate font-medium text-textStrong"] $ toHtml $ fromMaybe "Untitled conversation" c.title + span_ [class_ "truncate text-xs text-textWeak"] $ toHtml $ T.intercalate " · " $ catMaybes [Just $ T.take 12 c.key, viaNonEmpty head c.users, viaNonEmpty head c.services, Just $ toText (getDurationNSMS $ fromIntegral c.activeNs)] + signalsCell c = div_ [class_ "flex flex-wrap gap-1"] do + when (c.errors > 0 && null c.signals) $ tonePill_ ErrorBadge $ show c.errors <> " errors" + forM_ c.signals \s -> span_ [term "data-tippy-content" $ signalHelp s] $ tonePill_ (signalTone s) $ signalLabel s + + callColumns :: [Column Call] + callColumns = + [ col "Call" (\c -> div_ [class_ "flex min-w-0 flex-col"] $ span_ [class_ "truncate font-medium text-textStrong"] (toHtml c.name) >> span_ [class_ "truncate text-xs text-textWeak"] (toHtml $ fromMaybe "" c.preview)) & withAttrs [class_ "min-w-72"] + , col "Model" (chips . maybeToList . (.model)) & withAttrs [class_ "w-40"] + , col "Agent" (chips . maybeToList . (.agent)) & withAttrs [class_ "w-32 max-lg:hidden"] + , col "Tokens" (\c -> span_ [class_ "tabular-nums text-textWeak"] $ toHtml $ fromMaybe "—" $ tokenFlow c.inputTokens c.outputTokens) & withAttrs [class_ "w-28 text-right"] + , col "Cost" (\c -> span_ [class_ "tabular-nums text-textStrong"] $ toHtml $ maybe "—" GenAI.formatUsd c.cost) & withAttrs [class_ "w-20 text-right"] + , col "Latency" (\c -> span_ [class_ "tabular-nums"] $ toHtml $ getDurationNSMS $ fromIntegral c.duration) & withAttrs [class_ "w-20 text-right"] + , col "Status" (\c -> if c.failed then tonePill_ ErrorBadge "Error" else forM_ (viaNonEmpty head $ abnormal c) (tonePill_ WarningBadge . show)) & withAttrs [class_ "w-24"] + , col "Time" (localTimeFmt_ "MMM dd, HH:mm:ss" . (.timestamp)) & withAttrs [class_ "w-28 max-md:hidden"] + ] + + +signalTone :: Signal -> BadgeColor +signalTone = \case + (ToolError; ModelError) -> ErrorBadge + (Truncated; Filtered; Looping; ContextPressure) -> WarningBadge + + +chips :: [Text] -> Html () +chips xs = div_ [class_ "flex min-w-0 flex-wrap gap-1"] $ forM_ (take 2 xs) (span_ [class_ "truncate rounded bg-fillWeak px-1.5 py-0.5 text-xs text-textWeak"] . toHtml) >> when (length xs > 2) (span_ [class_ "text-xs text-textWeak"] $ toHtml $ "+" <> show @Text (length xs - 2)) + + +num :: Double -> Html () +num = span_ [class_ "tabular-nums text-textWeak"] . toHtml . prettyPrintCount . round + + +-- * Conversation detail + + +data ConversationDetail = ConversationDetail {pid :: Projects.ProjectId, key :: Text, window :: TimePicker.TimeWindow, summary :: Maybe Conversation, steps :: [Call]} + + +newtype ConversationGet = ConversationPage (PageCtx ConversationDetail) + deriving newtype (ToHtml) + + +conversationGetH :: Projects.ProjectId -> Text -> Maybe Text -> Maybe Text -> Maybe Text -> ATAuthCtx (RespHeaders ConversationGet) +conversationGetH pid key fromP toP sinceP = do + (_, _, bw) <- mkPageCtx pid + window <- mkWindow fromP toP sinceP + steps <- + reverse + <$> selectCalls + pid + window + True + [HI.sql| AND (context___trace_id = #{key} OR attributes -> 'gen_ai' -> 'conversation' ->> 'id' = #{key} OR context___trace_id IN ( + SELECT context___trace_id FROM otel_logs_and_spans + WHERE project_id = #{pid.toText} AND timestamp BETWEEN #{window.fromTime} AND #{window.toTime} AND context___trace_id <> '' + AND attributes -> 'gen_ai' -> 'conversation' ->> 'id' = #{key})) |] + let summary = viaNonEmpty head $ groupConversations $ V.fromList steps + addRespHeaders + $ ConversationPage + $ PageCtx + bw + { pageTitle = "LLM & Agents" + , pageTitleUrl = Just $ windowed window ((urls.traces pid).llmGetH Nothing Nothing Nothing Nothing) Nothing + , pageTitleSuffix = Just $ maybe key (T.take 60 . fromMaybe "Conversation" . (.title)) summary + , menuItem = Just "LLM & Agents" + , pageActions = Just $ TimePicker.liveDataControls_ Nothing window.currentRange Nothing TimePicker.RefreshOnly + } + ConversationDetail{pid, key, window, summary, steps} + + +instance ToHtml ConversationDetail where + toHtml = toHtmlRaw . conversationDetail_ + toHtmlRaw = toHtml + + +conversationDetail_ :: ConversationDetail -> Html () +conversationDetail_ d = div_ [class_ "mx-auto flex w-full max-w-6xl flex-col gap-4 p-4"] $ case d.summary of + Nothing -> callout_ WarningBadge Nothing "No spans for this conversation in the selected time range. Widen the range from the picker above." + Just c -> do + header_ [class_ "flex flex-col gap-2"] do + div_ [class_ "flex flex-wrap items-start gap-3"] do + h1_ [class_ "min-w-0 flex-1 text-lg font-semibold text-textStrong text-pretty max-sm:basis-full"] $ toHtml $ fromMaybe "Untitled conversation" c.title + askAi_ c + a_ ([class_ "btn btn-sm", href_ explorer] <> navTabAttrs) $ faSprite_ "explore" "regular" "h-3.5 w-3.5" >> "Open spans in Explorer" + div_ [class_ "flex flex-wrap gap-1.5"] do + forM_ c.agents $ metaChip_ "Agent" + forM_ c.models $ metaChip_ "Model" + forM_ (take 1 c.users) $ metaChip_ "User" + metaChip_ "Turns" $ show c.turns + metaChip_ "Calls" $ show c.llmCalls <> " model · " <> show c.toolCalls <> " tool" + metaChip_ "Tokens" $ prettyPrintCount $ round c.tokens + metaChip_ "Cost" $ maybe "unpriced" GenAI.formatUsd c.cost + metaChip_ "Active time" $ toText $ getDurationNSMS $ fromIntegral c.activeNs + unless (null c.signals) $ div_ [class_ "flex flex-col gap-1.5"] $ forM_ c.signals \s -> callout_ (signalTone s) Nothing $ strong_ (toHtml $ signalLabel s) >> " — " >> toHtml (signalHelp s) + div_ [class_ "group/ctab flex flex-col gap-3"] do + div_ [role_ "radiogroup", Aria.label_ "Conversation views", class_ "tabs tabs-box w-max"] do + label_ [class_ "tab has-[:checked]:tab-active"] $ input_ [type_ "radio", name_ "ctab", class_ "sr-only ctab-transcript", checked_] >> "Transcript" + label_ [class_ "tab has-[:checked]:tab-active"] $ input_ [type_ "radio", name_ "ctab", class_ "sr-only ctab-timeline"] >> "Timeline" + section_ [class_ "hidden group-has-[.ctab-transcript:checked]/ctab:block"] transcript_ + section_ [class_ "hidden group-has-[.ctab-timeline:checked]/ctab:block"] timeline_ + where + explorer = (urls.logExplorer d.pid).logExplorerGet (Just $ "attributes.gen_ai.conversation.id == \"" <> d.key <> "\" or trace_id == \"" <> d.key <> "\"") Nothing d.window.sinceQuery d.window.fromQuery d.window.toQuery (Just "spans") Nothing Nothing Nothing Nothing Nothing Nothing Nothing + + -- \| Hands the conversation to the assistant dock, which reads the page it was asked from. + askAi_ :: Conversation -> Html () + askAi_ c = + button_ + [ type_ "button" + , class_ "btn btn-sm btn-primary" + , hxPost_ $ urls.aiDockPost d.pid + , hxVals_ $ "js:{path: location.pathname + location.search, title: document.title, query: " <> decodeUtf8 (AE.encode prompt) <> "}" + , hxTarget_ "#ai-dock" + , hxSwap_ "outerMorph" + , [__|on htmx:after:request set #ai-dock-open.checked to true|] + ] + $ faSprite_ "sparkles" "regular" "h-3.5 w-3.5" + >> "Ask AI about this conversation" + where + prompt = "Analyse the agent conversation on this page (gen_ai.conversation.id or trace id " <> c.key <> "). Summarise what the user wanted, whether the agent achieved it, where it went wrong (tool errors, loops, truncation, wrong tool choice), and what drove the cost and latency. Query its spans if you need more than the page shows." :: Text + + -- \| Each message once: an LLM call's input repeats the history before it, so only messages + -- not already shown are rendered, then the call's reply with its model, latency and cost. + transcript_ :: Html () + transcript_ = div_ [class_ "flex flex-col gap-3"] do + unless (any (\s -> isJust (messages "input" s) || isJust (messages "output" s)) source) $ callout_ NeutralBadge Nothing "These spans carry no message content. Enable content capture in your instrumentation (OTEL_INSTRUMENTATION_GENAI_CAPTURE_CONTENT=SPAN_ONLY for OpenTelemetry GenAI) to see the transcript." + sequence_ rendered + where + calls = filter inference d.steps + source = if any (isJust . messages "input") calls then calls else filter ((== "invoke_agent") . (.operation)) d.steps + at c = genAIAt (unAesonTextMaybe c.attributes) + messages dir c = at c [dir, "messages"] >>= \case AE.Array ms -> Just (toList ms); _ -> Nothing + tools = M.fromList [(i, c) | c <- d.steps, Just (AE.String i) <- [at c ["tool", "call", "id"]]] + rendered = go HS.empty source + go _ [] = [] + go seen (s : rest) = + let fresh = filter (not . (`HS.member` seen)) (fromMaybe [] $ messages "input" s) + out = fromMaybe [] $ messages "output" s + seen' = foldr HS.insert seen (fresh <> out) + in (forM_ fresh (message_ d.pid tools Nothing) >> forM_ out (message_ d.pid tools (Just s))) : go seen' rest + + -- \| AI spans in execution order, with a bar on the conversation's own clock and the running cost. + timeline_ :: Html () + timeline_ = div_ [class_ "overflow-hidden rounded-lg border border-strokeWeak"] $ table_ [class_ "w-full text-sm"] do + thead_ [class_ "bg-fillWeaker text-xs text-textWeak"] $ tr_ do + th_ [class_ "px-3 py-1.5 text-left font-medium"] "Step" + th_ [class_ "w-1/3 px-3 py-1.5 text-left font-medium max-md:hidden"] "When" + th_ [class_ "px-3 py-1.5 text-right font-medium"] "Duration" + th_ [class_ "px-3 py-1.5 text-right font-medium"] "Tokens" + th_ [class_ "px-3 py-1.5 text-right font-medium"] "Cost · total" + tbody_ $ forM_ (zip d.steps running) \(c, total) -> do + let offset = pct (diffUTCTime c.timestamp start) + width = max 0.5 $ pct (fromIntegral c.duration / 1e9) + tr_ ([class_ "border-t border-strokeWeak hover:bg-fillHover"] <> maybe [] (\u -> drawerRowAttrs_ $ (urls.logExplorer d.pid).logExplorerItemDetailedGet u c.timestamp Nothing (Just "tab-genai") Nothing True) (UUID.fromText c.id)) do + td_ [class_ "px-3 py-1.5"] $ div_ [class_ "flex min-w-0 items-center gap-2"] do + faSprite_ (opIcon c.operation) "regular" "h-3 w-3 shrink-0 text-iconNeutral" + span_ [class_ "truncate text-textStrong"] $ toHtml c.name + when c.failed $ tonePill_ ErrorBadge "Error" + td_ [class_ "px-3 py-1.5 max-md:hidden"] $ div_ [class_ "relative h-2 rounded-sm bg-fillWeak"] $ div_ [class_ $ "absolute inset-y-0 rounded-sm " <> if c.failed then "bg-fillError-strong" else "bg-fillBrand-strong", style_ $ "left:" <> show offset <> "%;width:" <> show (min width (100 - offset)) <> "%"] "" + td_ [class_ "px-3 py-1.5 text-right tabular-nums text-textWeak"] $ toHtml $ getDurationNSMS $ fromIntegral c.duration + td_ [class_ "px-3 py-1.5 text-right tabular-nums text-textWeak"] $ toHtml $ fromMaybe "" $ tokenFlow c.inputTokens c.outputTokens + td_ [class_ "px-3 py-1.5 text-right tabular-nums text-textWeak"] $ toHtml $ maybe "" (\x -> GenAI.formatUsd x <> " · " <> GenAI.formatUsd total) c.cost + where + start = maybe d.window.fromTime (.timestamp) (viaNonEmpty head d.steps) + end = maybe start (F1.maximum . fmap (\s -> addUTCTime (fromIntegral s.duration / 1e9) s.timestamp)) (nonEmpty d.steps) + span' = max 0.001 $ diffUTCTime end start + pct x = realToFrac (100 * x / span') :: Double + running = drop 1 $ scanl (+) 0 $ map (fromMaybe 0 . (.cost)) d.steps + opIcon = \case + "execute_tool" -> "function" + "invoke_agent" -> "sparkles" + _ -> "comment" diff --git a/src/Pages/LogExplorer/LogItem.hs b/src/Pages/LogExplorer/LogItem.hs index b7512185a..5be243114 100644 --- a/src/Pages/LogExplorer/LogItem.hs +++ b/src/Pages/LogExplorer/LogItem.hs @@ -39,6 +39,7 @@ import Models.Telemetry.Telemetry (atMapText) import Models.Telemetry.Telemetry qualified as Telemetry import NeatInterpolation (text) import Pages.Components (BadgeColor (..), EmptyStateAction (..), EmptyStateCfg (..), EmptyStateSize (..), copySourceAttr_, curlCommand, dateTime, detailTab_, disclosure_, emptyState_, httpTab_, stackTrace_, tabPanel_, tonePill_) +import Pages.GenAIMessages qualified as LLM import Pkg.DeriveUtils (unAesonTextMaybe) import Pkg.ErrorFingerprint qualified as EF import Relude @@ -516,7 +517,8 @@ expandedItemView pid item aptSp selectedTabM = do detailTabs :: Projects.ProjectId -> Telemetry.OtelLogsAndSpans -> Maybe Telemetry.OtelLogsAndSpans -> [DetailTab] detailTabs pid item aptSp = catMaybes - [ tab + [ tab (LLM.isGenAISpan attrs) "tab-genai" "" "Conversation" "group-has-[.tab-genai:checked]/dtab:block" "genai-content" (LLM.genAISpanDetail_ pid attrs) + , tab ((isLog || isAlert) && isJust item.body) "tab-body" "" @@ -578,6 +580,7 @@ detailTabs pid item aptSp = isLog = item.kind == Just "log" isAlert = item.kind == Just "alert" isHttp = not isLog && isHttpSpan item + attrs = unAesonTextMaybe item.attributes events = fromMaybe AE.Null (unAesonTextMaybe item.events) spanErrors = if isLog then [] else getSpanErrors events tab shown marker cls label panelClass panelId content = DetailTab{marker, cls, label, panelClass, panelId, content} <$ guard shown diff --git a/src/Pages/Telemetry.hs b/src/Pages/Telemetry.hs index ee1b8d14d..ced179178 100644 --- a/src/Pages/Telemetry.hs +++ b/src/Pages/Telemetry.hs @@ -59,6 +59,7 @@ import NeatInterpolation (text) import Pages.BodyWrapper (BWConfig (..), PageCtx (..), mkPageCtx, navTabAttrs) import Pages.Components (EmptyStateAction (..), EmptyStateCfg (..), EmptyStateSize (..), explorerNavTabs_) import Pages.Components qualified as Components +import Pages.GenAIMessages qualified as GenAIMessages import Pages.LogExplorer.LogItem (getRequestDetails, getServiceColor, getServiceName, spanHasErrors) import Pages.LogExplorer.LogItem qualified as LogItem import Pkg.Components.ServiceMap (serviceMapPanel_) @@ -67,6 +68,7 @@ import Pkg.Components.Table qualified as Table import Pkg.Components.TimePicker qualified as TimePicker import Pkg.Components.Widget qualified as Widget import Pkg.DeriveUtils (UUIDId (..)) +import Pkg.GenAI qualified as GenAI import Relude hiding (ask) import Relude.Extra.Foldable1 (maximum1, minimum1) import System.Config (AuthContext (..), EnvConfig (..)) @@ -1288,6 +1290,8 @@ tracePage pid traceItem rawSpanRecords moreUrl = do stBox "Spans" (show (length spanRecords) <> foldMap (const "+") moreUrl) Nothing stBox "Errors" (show $ length $ V.filter (\s -> s.status == Just SSError) spanRecords) $ Just (faSprite_ "alert-triangle" "regular" "w-3 h-3 text-iconError") stBox "Duration" (toText $ getDurationNSMS traceItem.traceDurationNs) $ Just (faSprite_ "clock" "regular" "w-3 h-3 text-iconNeutral") + whenJust (viaNonEmpty sum $ mapMaybe (\sp -> GenAIMessages.genAINum sp.attributes ["usage", "cost"]) $ V.toList spanRecords) \cost -> + stBox "LLM cost" (GenAI.formatUsd cost) $ Just (faSprite_ "sparkles" "regular" "w-3 h-3 text-iconNeutral") whenJust moreUrl \url -> div_ [id_ "trace-load-more", class_ "flex items-center justify-between gap-2 rounded-lg border border-strokeWeak bg-fillWeaker px-3 py-1.5"] do span_ [class_ "text-xs text-textWeak"] "This trace is larger than one page. Search and the waterfall only cover what's loaded." @@ -1658,6 +1662,7 @@ buildSpanTree_ pid sp level scol = do do span_ [class_ "w-20 max-w-[30%] shrink-0 text-xs text-textWeak truncate"] $ toHtml sp.spanRecord.serviceName span_ [class_ "min-w-0 flex-1 text-xs text-textStrong truncate"] $ toHtml label + whenJust (GenAIMessages.genAIUsage sp.spanRecord.attributes) $ span_ [class_ "shrink-0 text-2xs tabular-nums text-textWeak max-md:hidden"] . toHtml template_ [data_ "trace-tooltip" ""] $ div_ [class_ "space-y-3"] do div_ [class_ "flex flex-wrap items-center gap-2"] do strong_ [class_ "min-w-0 flex-1 text-sm font-semibold break-normal [overflow-wrap:anywhere] text-textStrong"] $ toHtml sp.spanRecord.serviceName diff --git a/src/Pkg/AI.hs b/src/Pkg/AI.hs index 8ef819d50..eac64a3d8 100644 --- a/src/Pkg/AI.hs +++ b/src/Pkg/AI.hs @@ -112,6 +112,7 @@ import Deriving.Aeson qualified as DAE import Effectful (Eff, IOE, (:>)) import Effectful.Exception qualified as Exception import Effectful.Log qualified as Log +import Effectful.Reader.Static qualified as Reader import Effectful.Time qualified as Time import Hasql.Interpolate qualified as HI import Hasql.Transaction qualified as Tx @@ -140,6 +141,7 @@ import Network.URI qualified as URI import Network.Wreq qualified as Wreq import OpenAI.V1.Chat.Completions qualified as OpenAIV1 import OpenAI.V1.Tool qualified as OAITool +import OpenTelemetry.Attributes qualified as OA import Pkg.AgentCatalog (LocalOperation (..)) import Pkg.AgentCatalog qualified as Catalog import Pkg.Components.TimePicker (TimePicker (..), parseTimeRange) @@ -151,6 +153,7 @@ import Pkg.SchemaLearning (FacetData (..), FacetSummary (..), FacetValue (..)) import Pkg.Slack qualified as Slack import Relude import System.Config qualified as Config +import System.Tracing (SpanKind (Internal), SpanStatus (Error), Tracing, setStatus, withSpanKind) import System.Types (DB, ProjectToolCtx) import UnliftIO.Exception (throwIO, withException) import Utils (encodeText, formatUTCMicros, hostPath, nonEmptyT, unwrapJsonPrimValue) @@ -1416,8 +1419,8 @@ runAgenticChatWithHistory suppliedConfig requestedInput model apiKey = do -- | Each attempt gets its own journal; a retry never overwrites partial evidence. -- Persisted Slack turns resume their checkpoint while each worker attempt retains its own activity log. -withInvestigationJournal :: (DB es, IOE :> es, UUIDEff :> es, W.HTTP :> es) => AgenticConfig -> Text -> Text -> (Maybe Investigations.Scope -> Eff es (Either Text a)) -> Eff es (Either Text a) -withInvestigationJournal config userQuery model action = do +withInvestigationJournal :: (DB es, IOE :> es, Tracing :> es, UUIDEff :> es, W.HTTP :> es) => AgenticConfig -> Text -> Text -> (Maybe Investigations.Scope -> Eff es (Either Text a)) -> Eff es (Either Text a) +withInvestigationJournal config userQuery model action = withSpanKind Internal "invoke_agent monoscope" agentAttrs \_ -> do void $ requireConfigAccess config case config.access of SlackInvestigationAccess run -> do @@ -1432,6 +1435,10 @@ withInvestigationJournal config userQuery model action = do Investigations.recordEvent scope $ either (const Investigations.InvestigationFailed) (const Investigations.InvestigationFinished) result pure result (MemberAccess{}; ServiceAccess; SlackAccess{}; SlackCommandAccess{}) -> action Nothing + where + agentAttrs = + [("gen_ai.operation.name", "invoke_agent"), ("gen_ai.agent.name", "monoscope"), ("gen_ai.provider.name", "openai"), ("gen_ai.request.model", OA.toAttribute model), ("monoscope.project_id", OA.toAttribute config.projectId.toText)] + <> [("gen_ai.conversation.id", OA.toAttribute convId.toText) | Just convId <- [config.conversationId]] -- | Save each completed decision/read before advancing. Pending read tools may @@ -1508,7 +1515,13 @@ runAgenticLoopRaw turn journal config apiKey checkpoint = do tool : remaining -> do Log.logTrace "AI requesting tool" (AE.object ["iteration" AE..= step.iteration, "tool" AE..= LLM.toolFunctionName (LLM.toolCallFunction tool)]) record $ Investigations.ToolStarted step.iteration tool - outcome <- executeToolAt turn config{timeRange = step.timeRange} tool + let callFunction = LLM.toolCallFunction tool + capture <- ELLM.capturesContent . (.env.otelInstrumentationGenaiCaptureContent) <$> Reader.ask @Config.AuthContext + outcome <- withSpanKind Internal ("execute_tool " <> LLM.toolFunctionName callFunction) ([("gen_ai.operation.name", "execute_tool"), ("gen_ai.tool.name", OA.toAttribute $ LLM.toolFunctionName callFunction), ("gen_ai.tool.call.id", OA.toAttribute $ LLM.toolCallId tool)] <> [("gen_ai.tool.call.arguments", OA.toAttribute $ encodeText $ LLM.toolFunctionArguments callFunction) | capture]) \sp -> do + outcome <- executeToolAt turn config{timeRange = step.timeRange} tool + outcome <$ case outcome of + Investigations.RefusedResult{} -> setStatus sp (Error "refused by review") + (Investigations.RecordedResult{}; Investigations.UncertainResult{}; Investigations.WaitingResult{}) -> pass let result = Investigations.operationResponse config.projectId outcome attempted = completed <> [(tool, result)] next <- case outcome of diff --git a/src/Pkg/GenAI.hs b/src/Pkg/GenAI.hs new file mode 100644 index 000000000..bed98bf6a --- /dev/null +++ b/src/Pkg/GenAI.hs @@ -0,0 +1,433 @@ +-- | One vocabulary for LLM telemetry. Every dialect in the wild (OpenInference, +-- OpenLLMetry/Traceloop, Vercel AI SDK, Langfuse, Claude Code) is mapped onto the +-- OpenTelemetry GenAI conventions at ingest, additively, and priced spans get +-- @gen_ai.usage.cost@ (USD). See docs/llm-observability.md. +module Pkg.GenAI ( + normalizeGenAI, + canonicalMessages, + ModelPrice (..), + Usage (..), + lookupPrice, + normaliseModel, + estimateCost, + isInferenceOp, + formatUsd, + FinishReasons (..), + Abnormal (..), + abnormalFinish, +) where + +import Data.Aeson ((.=)) +import Data.Aeson qualified as AE +import Data.Aeson.Key qualified as AEK +import Data.Aeson.KeyMap qualified as KM +import Data.Char (isDigit) +import Data.FileEmbed (embedFile) +import Data.HashMap.Strict qualified as HM +import Data.IntMap.Strict qualified as IntMap +import Data.List (lookup) +import Data.Map.Strict qualified as M +import Data.Scientific (toRealFloat) +import Data.Text qualified as T +import Data.Vector qualified as V +import Numeric (showFFloat) +import Relude + + +-- | Prices are USD per million tokens; a missing cache rate falls back to the input rate. +data ModelPrice = ModelPrice {input :: Double, output :: Double, cacheRead :: Maybe Double, cacheWrite :: Maybe Double, contextWindow :: Maybe Int} + deriving stock (Eq, Show) + + +data Usage = Usage {input :: Double, output :: Double, cacheRead :: Double, cacheWrite :: Double} + deriving stock (Eq, Show) + + +-- | Generated by scripts/update-model-prices.py from LiteLLM's MIT price table. +-- +-- >>> HM.size prices > 1000 +-- True +prices :: HM.HashMap Text ModelPrice +prices = maybe mempty (fmap price) $ AE.decodeStrict $(embedFile "static/model_prices.json") + where + price (i, o, r, w, c) = ModelPrice i o r w c :: ModelPrice + + +-- | The id the price table is keyed by: no provider path, region or vendor prefix, +-- version pin or Bedrock revision. +-- +-- >>> map normaliseModel ["openai/gpt-4o", "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-3-5-sonnet@20240620", "models/gemini-2.5-pro", "GPT-4o"] +-- ["gpt-4o","claude-sonnet-4-5-20250929","claude-3-5-sonnet","gemini-2.5-pro","gpt-4o"] +normaliseModel :: Text -> Text +normaliseModel raw = unpin $ dropVendor $ snd $ T.breakOnEnd "/" $ T.toLower raw + where + dropVendor m = case break (\s -> any (`T.isPrefixOf` s) families) (T.splitOn "." m) of + (_ : _, family@(_ : _)) -> T.intercalate "." family + _ -> m + families = ["claude", "llama", "mistral", "nova", "titan", "command", "jamba"] + -- Bedrock pins a revision as "-v1:0"; elsewhere "-v3" is part of the name. + unpin m = case T.break (== ':') m of + (pre, pin) | not (T.null pin), (base, rev) <- T.breakOnEnd "-v" pre, not (T.null base), not (T.null rev), T.all isDigit rev -> T.dropEnd 2 base + (pre, _) -> T.takeWhile (/= '@') pre + + +-- | Exact id, then without a date pin, then the longest known family prefix, then the +-- nearest dated release of a bare alias. +-- +-- >>> (.input) <$> lookupPrice "gpt-4o-2024-08-06" +-- Just 2.5 +-- >>> (.input) <$> lookupPrice "anthropic.claude-opus-4-1-20250805-v1:0" +-- Just 15.0 +-- >>> (.input) <$> lookupPrice "claude-opus-4-1" +-- Just 15.0 +-- >>> lookupPrice "my-finetune" +-- Nothing +lookupPrice :: Text -> Maybe ModelPrice +lookupPrice raw = asum (map (`HM.lookup` prices) (m : undated : families)) <|> dated + where + m = normaliseModel raw + parts = T.splitOn "-" m + undated = T.intercalate "-" $ reverse $ dropWhile (\p -> T.all isDigit p && T.length p `elem` [2, 4, 8]) $ reverse parts + families = [T.intercalate "-" (take n parts) | n <- [length parts - 1, length parts - 2 .. 1]] + dated = M.lookupGE (m <> "-") pricesByKey >>= \(k, p) -> p <$ guard ((m <> "-") `T.isPrefixOf` k) + + +-- | Ordered for the dated-release lookup: a bare alias sorts just before its releases. +pricesByKey :: M.Map Text ModelPrice +pricesByKey = M.fromList $ HM.toList prices + + +-- | Totals per the GenAI conventions include cached tokens; Anthropic-native counts +-- exclude them, which shows as cache tokens exceeding the input total. +-- +-- >>> let sonnet = ModelPrice 3 15 (Just 0.3) (Just 3.75) Nothing +-- >>> estimateCost sonnet (Usage 1000000 0 0 0) +-- 3.0 +-- >>> estimateCost sonnet (Usage 1000000 0 800000 0) +-- 0.84 +-- >>> estimateCost sonnet (Usage 10 0 1000000 0) +-- 0.30003 +-- >>> estimateCost (ModelPrice 2 8 Nothing Nothing Nothing) (Usage 1000 500 500 0) +-- 6.0e-3 +estimateCost :: ModelPrice -> Usage -> Double +estimateCost p u = max 0 $ (uncached * p.input + u.cacheRead * fromMaybe p.input p.cacheRead + u.cacheWrite * fromMaybe p.input p.cacheWrite + u.output * p.output) / 1e6 + where + cached = u.cacheRead + u.cacheWrite + uncached = if u.input >= cached then u.input - cached else u.input + + +-- | Sub-cent amounts keep four decimals: a single call usually costs fractions of a cent. +-- +-- >>> map formatUsd [0, 0.00004, 0.0042, 0.42, 12.3456] +-- ["$0","<$0.0001","$0.0042","$0.42","$12.35"] +formatUsd :: Double -> Text +formatUsd c + | c <= 0 = "$0" + | c < 0.0001 = "<$0.0001" + | otherwise = "$" <> toText (showFFloat (Just $ if c < 0.01 then 4 else 2) c "") + + +-- | @gen_ai.response.finish_reasons@: an array by the conventions, a single string from some producers. +newtype FinishReasons = FinishReasons [Text] + deriving stock (Eq, Show) + + +instance AE.FromJSON FinishReasons where + parseJSON v = FinishReasons <$> (AE.parseJSON v <|> (pure <$> AE.parseJSON v)) + + +data Abnormal = Truncated | Filtered + deriving stock (Eq, Show) + + +-- | A finish reason that means the reply is incomplete, across providers' spellings. +-- +-- >>> map abnormalFinish ["length", "MAX_TOKENS", "content_filter", "SAFETY", "stop", "end_turn"] +-- [Just Truncated,Just Truncated,Just Filtered,Just Filtered,Nothing,Nothing] +abnormalFinish :: Text -> Maybe Abnormal +abnormalFinish r + | r `elem` ["length", "max_tokens", "MAX_TOKENS"] = Just Truncated + | r `elem` ["content_filter", "refusal", "SAFETY"] = Just Filtered + | otherwise = Nothing + + +isInferenceOp :: Text -> Bool +isInferenceOp = (`elem` ["chat", "text_completion", "generate_content", "embeddings"]) + + +-- | Canonical GenAI keys from whichever dialect the producer speaks. Originals are kept, +-- except indexed message keys, which are folded into one messages array. +-- +-- OpenInference, with cost estimated from the price table: +-- +-- >>> let oi = normalizeGenAI [("openinference.span.kind", "LLM"), ("llm.model_name", "gpt-4o"), ("llm.token_count.prompt", AE.Number 1000), ("llm.token_count.completion", AE.Number 100), ("llm.input_messages.0.message.role", "user"), ("llm.input_messages.0.message.content", "hi"), ("session.id", "s1")] +-- >>> mapMaybe (`lookup` oi) ["gen_ai.operation.name", "gen_ai.request.model", "gen_ai.usage.input_tokens", "gen_ai.usage.cost", "gen_ai.usage.cost_source", "gen_ai.conversation.id"] +-- [String "chat",String "gpt-4o",Number 1000.0,Number 3.5e-3,String "estimated",String "s1"] +-- >>> lookup "gen_ai.input.messages" oi +-- Just (Array [Object (fromList [("parts",Array [Object (fromList [("content",String "hi"),("type",String "text")])]),("role",String "user")])]) +-- >>> any (T.isPrefixOf "llm.input_messages" . fst) oi +-- False +-- >>> lookup "gen_ai.input.preview" oi +-- Just (String "hi") +-- +-- OpenLLMetry's indexed prompt with a tool call; the old token names; JSON-string messages: +-- +-- >>> let ol = normalizeGenAI [("gen_ai.system", "openai"), ("gen_ai.request.model", "gpt-4o-mini"), ("llm.request.type", "chat"), ("gen_ai.usage.prompt_tokens", AE.Number 10), ("gen_ai.completion.0.role", "assistant"), ("gen_ai.completion.0.tool_calls.0.name", "search"), ("gen_ai.completion.0.tool_calls.0.arguments", "{\"q\":1}"), ("gen_ai.completion.0.finish_reason", "tool_calls")] +-- >>> mapMaybe (`lookup` ol) ["gen_ai.provider.name", "gen_ai.usage.input_tokens", "gen_ai.response.finish_reasons"] +-- [String "openai",Number 10.0,Array [String "tool_calls"]] +-- >>> lookup "gen_ai.output.messages" ol +-- Just (Array [Object (fromList [("finish_reason",String "tool_calls"),("parts",Array [Object (fromList [("arguments",String "{\"q\":1}"),("id",Null),("name",String "search"),("type",String "tool_call")])]),("role",String "assistant")])]) +-- >>> let canonical = normalizeGenAI [("gen_ai.operation.name", "chat"), ("gen_ai.input.messages", "[{\"role\":\"user\",\"content\":\"yo\"}]")] +-- >>> lookup "gen_ai.input.preview" canonical +-- Just (String "yo") +-- >>> lookup "gen_ai.input.messages" canonical +-- Just (Array [Object (fromList [("parts",Array [Object (fromList [("content",String "yo"),("type",String "text")])]),("role",String "user")])]) +-- +-- Claude Code log events: string numbers, exclusive input counts made inclusive, its own cost: +-- +-- >>> let cc = normalizeGenAI [("event.name", "api_request"), ("prompt.id", "98ec"), ("model", "claude-sonnet-4-5"), ("input_tokens", "10"), ("cache_read_tokens", "5000"), ("output_tokens", "20"), ("cost_usd", "0.0021"), ("session.id", "abc")] +-- >>> mapMaybe (`lookup` cc) ["gen_ai.operation.name", "gen_ai.provider.name", "gen_ai.usage.input_tokens", "gen_ai.usage.cost", "gen_ai.usage.cost_source", "gen_ai.conversation.id"] +-- [String "chat",String "anthropic",Number 5010.0,Number 2.1e-3,String "reported",String "abc"] +-- +-- A real @claude -p@ user_prompt record becomes the turn's user message: +-- +-- >>> lookup "gen_ai.input.messages" $ normalizeGenAI [("event.name", "user_prompt"), ("event.sequence", AE.Number 22), ("prompt.id", "98ec"), ("prompt_length", "30"), ("prompt", "Reply with the single word: hi"), ("session.id", "1381")] +-- Just (Array [Object (fromList [("parts",Array [Object (fromList [("content",String "Reply with the single word: hi"),("type",String "text")])]),("role",String "user")])]) +-- +-- Vercel's outer span is an agent run: its usage aggregates its steps, so it is not priced. +-- +-- >>> let v = normalizeGenAI [("ai.operationId", "ai.generateText"), ("ai.model.id", "gpt-4o"), ("ai.usage.promptTokens", AE.Number 5)] +-- >>> mapMaybe (`lookup` v) ["gen_ai.operation.name", "gen_ai.usage.input_tokens", "gen_ai.usage.cost"] +-- [String "invoke_agent"] +-- +-- Records with no GenAI marker pass through untouched, even with a session, a bare model key or +-- an event name Claude Code also uses: +-- +-- >>> normalizeGenAI [("session.id", "s"), ("model", "x"), ("http.request.method", "GET")] +-- [("session.id",String "s"),("model",String "x"),("http.request.method",String "GET")] +-- >>> normalizeGenAI [("event.name", "api_error"), ("session.id", "s")] +-- [("event.name",String "api_error"),("session.id",String "s")] +-- +-- A model with no operation is a chat call only when it reports tokens: +-- +-- >>> lookup "gen_ai.operation.name" <$> map normalizeGenAI [[("gen_ai.request.model", "gpt-4o")], [("gen_ai.request.model", "gpt-4o"), ("gen_ai.usage.input_tokens", AE.Number 3)]] +-- [Nothing,Just (String "chat")] +normalizeGenAI :: [(Text, AE.Value)] -> [(Text, AE.Value)] +normalizeGenAI kvs + | isNothing ccEvent && not (any (\(k, _) -> any (`T.isPrefixOf` k) markers) kvs) = kvs + | otherwise = [(k, passThrough k v) | (k, v) <- kvs, not (folded k)] <> derived + where + markers = ["gen_ai.", "llm.", "ai.", "openinference.", "traceloop.", "langfuse."] + -- The guard above runs on every attribute set ingested, so it must not build the map. + -- These event names are common elsewhere; Claude Code also stamps every event with prompt.id. + ccEvent = guard (isJust $ lookup "prompt.id" kvs) *> lookup "event.name" kvs >>= str >>= \e -> find (== fromMaybe e (T.stripPrefix "claude_code." e)) ["api_request", "api_error", "tool_result", "user_prompt"] + m = HM.fromList kvs + pick k = HM.lookup k m + firstOf = asum . map pick + text k = pick k >>= str + cc k = guard (isJust ccEvent) *> pick k + + op = + text "gen_ai.operation.name" + <|> (text "openinference.span.kind" >>= (`lookup` [("LLM", "chat"), ("TOOL", "execute_tool"), ("AGENT", "invoke_agent"), ("CHAIN", "invoke_workflow"), ("EMBEDDING", "embeddings"), ("RETRIEVER", "retrieval"), ("RERANKER", "rerank"), ("GUARDRAIL", "guardrail"), ("EVALUATOR", "evaluate")])) + <|> (text "llm.request.type" >>= (`lookup` [("chat", "chat"), ("completion", "text_completion"), ("embedding", "embeddings"), ("rerank", "rerank")])) + <|> (text "traceloop.span.kind" >>= (`lookup` [("workflow", "invoke_workflow"), ("agent", "invoke_agent"), ("tool", "execute_tool"), ("task", "task")])) + <|> (text "ai.operationId" >>= vercelOp) + <|> (ccEvent >>= (`lookup` [("api_request", "chat"), ("api_error", "chat"), ("tool_result", "execute_tool"), ("user_prompt", "user_prompt")])) + -- Raw keys, not the derived counts: those are guarded on this very operation. + <|> ("chat" <$ guard (isJust model && isJust (firstOf inputTokenKeys))) + vercelOp o + | any (`T.isSuffixOf` o) [".doGenerate", ".doStream"] = Just "chat" + | ".doEmbed" `T.isSuffixOf` o = Just "embeddings" + | o == "ai.toolCall" = Just "execute_tool" + | "ai." `T.isPrefixOf` o = Just "invoke_agent" + | otherwise = Nothing + inference = maybe False isInferenceOp op + tool = op == Just "execute_tool" + + model = firstOf ["gen_ai.request.model", "llm.request.model_name", "llm.model_name", "ai.model.id", "langfuse.observation.model.name"] <|> cc "model" + responseModel = firstOf ["gen_ai.response.model", "llm.response.model_name", "ai.response.model"] + tokens keys = guard inference *> (firstOf keys >>= num) + inputTokenKeys = ["gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens", "llm.token_count.prompt", "ai.usage.promptTokens", "ai.usage.inputTokens"] + inputTokens = tokens inputTokenKeys + outputTokens = tokens ["gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens", "llm.token_count.completion", "ai.usage.completionTokens", "ai.usage.outputTokens"] + cacheRead = tokens ["gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.cache_read_input_tokens", "llm.token_count.prompt_details.cache_read"] + cacheWrite = tokens ["gen_ai.usage.cache_write.input_tokens", "gen_ai.usage.cache_creation.input_tokens", "gen_ai.usage.cache_creation_input_tokens", "llm.token_count.prompt_details.cache_write"] + ccTokens k = guard inference *> (cc k >>= num) + -- Claude Code counts input exclusive of the cache; the conventions count it inclusive. + usage = case ccEvent of + Just _ -> + let (i, r, w) = (ccTokens "input_tokens", ccTokens "cache_read_tokens", ccTokens "cache_creation_tokens") + in ((+ sum (catMaybes [r, w])) <$> i, ccTokens "output_tokens", r, w) + Nothing -> (inputTokens, outputTokens, cacheRead, cacheWrite) + (inT, outT, readT, writeT) = usage + reported = guard inference *> (firstOf ["gen_ai.usage.cost", "llm.cost.total", "gen_ai.cost.total_tokens"] <|> cc "cost_usd") >>= num + cost = (,"reported" :: Text) <$> reported <|> (,"estimated") <$> estimated + estimated = do + price <- guard inference *> ((model <|> responseModel) >>= str >>= lookupPrice) + guard $ isJust inT || isJust outT + pure $ estimateCost price $ Usage (fromMaybe 0 inT) (fromMaybe 0 outT) (fromMaybe 0 readT) (fromMaybe 0 writeT) + + inputMessages = indexedMessages "llm.input_messages." "message." <|> indexedMessages "gen_ai.prompt." "" <|> (pick "ai.prompt.messages" >>= json) <|> valueMessages "user" "input.value" <|> (single "user" <$> cc "prompt") + outputMessages = indexedMessages "llm.output_messages." "message." <|> indexedMessages "gen_ai.completion." "" <|> vercelOutput <|> valueMessages "assistant" "output.value" + valueMessages role k = guard (op == Just "chat") *> pick k <&> \v -> fromMaybe (single role v) (json v >>= messagesOf) + single :: Text -> AE.Value -> AE.Value + single role content = AE.Array [AE.object ["role" .= role, "content" .= content]] + messagesOf v = case v of + AE.Array _ -> Just v + AE.Object o -> KM.lookup "messages" o + _ -> Nothing + vercelOutput = do + let calls = maybe [] toList (pick "ai.response.toolCalls" >>= json >>= arr) + guard $ isJust (pick "ai.response.text") || not (null calls) + pure $ AE.Array [AE.object ["role" .= ("assistant" :: Text), "content" .= pick "ai.response.text", "tool_calls" .= map vercelCall calls]] + vercelCall c = AE.object ["id" .= field ["toolCallId"] c, "function" .= AE.object ["name" .= field ["toolName"] c, "arguments" .= field ["args", "input"] c]] + + -- OpenInference nests each message under "message."; OpenLLMetry does not. + indexedMessages prefix inner = case indexed prefix kvs of + [] -> Nothing + msgs -> Just $ AE.Array $ V.fromList $ map (indexedMessage inner) msgs + indexedMessage inner mm = + let at k = HM.lookup (inner <> k) mm + calls = [AE.object ["id" .= (HM.lookup "id" c <|> HM.lookup "tool_call.id" c), "function" .= AE.object ["name" .= (HM.lookup "name" c <|> HM.lookup "tool_call.function.name" c), "arguments" .= (HM.lookup "arguments" c <|> HM.lookup "tool_call.function.arguments" c)]] | c <- indexed (inner <> "tool_calls.") (HM.toList mm)] + in AE.object $ ["role" .= at "role", "content" .= at "content", "tool_call_id" .= at "tool_call_id", "finish_reason" .= at "finish_reason"] <> ["tool_calls" .= calls | not (null calls)] + -- Message content is the heaviest payload; once folded into the canonical arrays its + -- source keys go, as do indexed keys, which would otherwise nest as {"0": …} objects. + folded k = + any (\p -> maybe False (isDigit . fst) (T.stripPrefix p k >>= T.uncons)) ["llm.input_messages.", "llm.output_messages.", "gen_ai.prompt.", "gen_ai.completion."] + || (k `elem` ["ai.prompt.messages", "input.value"] && emitted "gen_ai.input.messages" inputMessages) + || (k `elem` ["ai.response.text", "ai.response.toolCalls", "output.value"] && emitted "gen_ai.output.messages" outputMessages) + emitted k v = not (HM.member k m) && isJust v + -- Message bodies are the heaviest values: decode each array once, shared by its emit and the preview. + givenInput = parsed "gen_ai.input.messages" <$> pick "gen_ai.input.messages" + derivedInput = canonicalMessages <$> inputMessages + passThrough k v = if k == "gen_ai.input.messages" then fromMaybe v givenInput else parsed k v + parsed k v + | k `elem` ["gen_ai.input.messages", "gen_ai.output.messages"] = canonicalMessages $ fromMaybe v (json v) + | k `elem` ["gen_ai.system_instructions", "gen_ai.tool.definitions"] = fromMaybe v (json v) + | otherwise = v + + finishReasons = firstOf ["gen_ai.response.finish_reasons", "llm.finish_reason", "llm.response.finish_reason", "ai.response.finishReason", "gen_ai.completion.0.finish_reason"] <|> cc "stop_reason" + asList = \case + v@(AE.Array _) -> v + v -> AE.Array [v] + conversation = firstOf ["gen_ai.conversation.id", "langfuse.session.id", "traceloop.association.properties.session_id", "ai.telemetry.metadata.sessionId", "ai.telemetry.metadata.conversationId", "ai.telemetry.metadata.threadId"] <|> (guard (isJust op) *> pick "session.id") + provider = (pick "gen_ai.system" <|> firstOf ["llm.provider", "llm.system"] <|> (text "ai.model.provider" <&> AE.String . T.takeWhile (/= '.'))) <|> ("anthropic" <$ ccEvent) + intV = AE.Number . fromInteger . round + + derived = + catMaybes + [ emit "gen_ai.operation.name" (AE.String <$> op) + , emit "gen_ai.provider.name" provider + , emit "gen_ai.request.model" model + , emit "gen_ai.response.model" responseModel + , emitTokens "gen_ai.usage.input_tokens" inT + , emitTokens "gen_ai.usage.output_tokens" outT + , emitTokens "gen_ai.usage.cache_read.input_tokens" readT + , emitTokens "gen_ai.usage.cache_write.input_tokens" writeT + , emit "gen_ai.usage.reasoning.output_tokens" (intV <$> tokens ["gen_ai.usage.reasoning_tokens", "llm.token_count.completion_details.reasoning"]) + , emit "gen_ai.usage.cost" (AE.toJSON . fst <$> cost) + , emit "gen_ai.usage.cost_source" (AE.String . snd <$> cost) + , emit "gen_ai.input.messages" derivedInput + , emit "gen_ai.output.messages" (canonicalMessages <$> outputMessages) + , emit "gen_ai.input.preview" (AE.String <$> ((givenInput <|> derivedInput) >>= latestUserText)) + , emit "gen_ai.response.finish_reasons" (asList <$> finishReasons) + , emit "gen_ai.conversation.id" conversation + , emit "gen_ai.agent.name" (firstOf ["agent.name", "traceloop.workflow.name", "ai.telemetry.functionId"]) + , emit "gen_ai.tool.name" (guard tool *> (firstOf ["tool.name", "ai.toolCall.name", "traceloop.entity.name"] <|> cc "tool_name")) + , emit "gen_ai.tool.call.id" (guard tool *> (firstOf ["tool_call.id", "ai.toolCall.id"] <|> cc "tool_use_id")) + , emit "gen_ai.tool.call.arguments" (guard tool *> (firstOf ["tool.parameters", "ai.toolCall.args", "ai.toolCall.input", "traceloop.entity.input", "input.value"] <|> cc "tool_parameters")) + , emit "gen_ai.tool.call.result" (guard tool *> firstOf ["ai.toolCall.result", "ai.toolCall.output", "traceloop.entity.output", "output.value"]) + , emit "user.id" (firstOf ["langfuse.user.id", "traceloop.association.properties.user_id", "ai.telemetry.metadata.userId"]) + ] + emit k v = guard (not $ HM.member k m) *> ((k,) <$> v) + -- Claude Code's counts replace their canonical twins, which it never sends. + emitTokens k v = (k,) . intV <$> v <* guard (isJust ccEvent || not (HM.member k m)) + + +-- | Messages in the conventions' parts shape, from OpenAI-, Anthropic-, Vercel- or +-- LangChain-shaped arrays. Already-canonical messages pass through. +-- +-- >>> canonicalMessages $ AE.Array [AE.object ["role" .= ("tool" :: Text), "tool_call_id" .= ("c1" :: Text), "content" .= ("42" :: Text)]] +-- Array [Object (fromList [("parts",Array [Object (fromList [("id",String "c1"),("response",String "42"),("type",String "tool_call_response")])]),("role",String "tool")])] +-- >>> canonicalMessages $ AE.Array [AE.object ["role" .= ("assistant" :: Text), "content" .= ([AE.object ["type" .= ("tool_use" :: Text), "id" .= ("t" :: Text), "name" .= ("ls" :: Text), "input" .= AE.object []]] :: [AE.Value])]] +-- Array [Object (fromList [("parts",Array [Object (fromList [("arguments",Object (fromList [])),("id",String "t"),("name",String "ls"),("type",String "tool_call")])]),("role",String "assistant")])] +canonicalMessages :: AE.Value -> AE.Value +canonicalMessages = \case + AE.Array msgs -> AE.Array $ V.map message msgs + v -> v + where + message = \case + AE.Object o | KM.member "parts" o -> AE.Object o + AE.Object o -> + let role = maybe "user" (\r -> fromMaybe r (lookup r [("human", "user"), ("ai", "assistant"), ("model", "assistant")])) (KM.lookup "role" o >>= str) + responses = [part "tool_call_response" ["id" .= i, "response" .= KM.lookup "content" o] | Just i <- [KM.lookup "tool_call_id" o], i /= AE.Null] + content = if null responses then foldMap contentParts (KM.lookup "content" o) else [] + calls = [part "tool_call" ["id" .= field ["id"] c, "name" .= (field ["function"] c >>= field ["name"]), "arguments" .= (field ["function"] c >>= field ["arguments"])] | c <- maybe [] toList (KM.lookup "tool_calls" o >>= arr)] + in AE.object $ ["role" .= role, "parts" .= (responses <> content <> calls)] <> ["finish_reason" .= f | Just f <- [KM.lookup "finish_reason" o], f /= AE.Null] + v -> AE.object ["role" .= ("user" :: Text), "parts" .= contentParts v] + contentParts = \case + AE.Null -> [] + AE.Array ps -> map contentPart (toList ps) + v -> [part "text" ["content" .= v]] + contentPart p = case field ["type"] p >>= str of + Just t | t `elem` ["text", "input_text", "output_text"] -> part "text" ["content" .= field ["text", "content"] p] + Just t | t `elem` ["tool-call", "tool_use", "function_call"] -> part "tool_call" ["id" .= field ["toolCallId", "id", "call_id"] p, "name" .= field ["toolName", "name"] p, "arguments" .= field ["args", "input", "arguments"] p] + Just t | t `elem` ["tool-result", "tool_result", "function_call_output"] -> part "tool_call_response" ["id" .= field ["toolCallId", "tool_use_id", "call_id"] p, "response" .= field ["result", "output", "content"] p] + Just t | t `elem` ["reasoning", "thinking"] -> part "reasoning" ["content" .= field ["text", "thinking", "content"] p] + _ -> p + part :: Text -> [(AEK.Key, AE.Value)] -> AE.Value + part t kvs = AE.object $ ("type" .= t) : kvs + + +-- | The turn's question, for lists and search: the last user text in canonical messages. +latestUserText :: AE.Value -> Maybe Text +latestUserText msgs = listToMaybe [T.take 200 t | m <- reverse (maybe [] toList (arr msgs)), field ["role"] m == Just "user", p <- maybe [] toList (field ["parts"] m >>= arr), field ["type"] p == Just "text", Just (AE.String t) <- [field ["content"] p]] + + +-- | Groups @prefix.N.rest@ keys by N, in index order. +indexed :: Text -> [(Text, AE.Value)] -> [HM.HashMap Text AE.Value] +indexed prefix kvs = + IntMap.elems + $ IntMap.fromListWith + (<>) + [ (n, one (T.drop 1 rest, v)) + | (k, v) <- kvs + , Just r <- [T.stripPrefix prefix k] + , let (ix, rest) = T.breakOn "." r + , not (T.null rest) + , Just n <- [readMaybe (toString ix)] + ] + + +field :: [Text] -> AE.Value -> Maybe AE.Value +field ks = \case + AE.Object o -> asum [KM.lookup (AEK.fromText k) o | k <- ks] >>= \v -> v <$ guard (v /= AE.Null) + _ -> Nothing + + +str :: AE.Value -> Maybe Text +str = \case + AE.String s -> Just s + _ -> Nothing + + +arr :: AE.Value -> Maybe (V.Vector AE.Value) +arr = \case + AE.Array a -> Just a + _ -> Nothing + + +num :: AE.Value -> Maybe Double +num = \case + AE.Number n -> Just $ toRealFloat n + AE.String s -> readMaybe $ toString s + _ -> Nothing + + +-- | Producers send structured attributes as JSON strings. +json :: AE.Value -> Maybe AE.Value +json = \case + AE.String s | Just (c, _) <- T.uncons (T.stripStart s), c `elem` ['[', '{'] -> AE.decodeStrict $ encodeUtf8 s + AE.String _ -> Nothing + v -> Just v diff --git a/src/Pkg/TestUtils.hs b/src/Pkg/TestUtils.hs index ae8d9fbd2..96e07578b 100644 --- a/src/Pkg/TestUtils.hs +++ b/src/Pkg/TestUtils.hs @@ -100,9 +100,11 @@ module Pkg.TestUtils ( ingestLog, ingestLogWithResource, ingestErrorLog, + ingestLogRecord, ingestTrace, ingestSpanLinked, ingestSpanReq, + withOwnTelemetry, withSpanKind, withSpanStatus, withTraceId, @@ -167,7 +169,7 @@ import Data.Generics.Product.Fields (field) import Data.GeoIP2 qualified as GeoIP2 import Data.HashMap.Strict qualified as HM import Data.Pool (Pool, defaultPoolConfig, destroyAllResources, newPool, setNumStripes, withResource) -import Data.ProtoLens (defMessage) +import Data.ProtoLens (decodeMessage, defMessage, encodeMessage) import Data.Text qualified as T import Data.Time (NominalDiffTime, UTCTime, ZonedTime, getCurrentTime, zonedTimeToUTC) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) @@ -219,8 +221,11 @@ import Network.Minio qualified as Minio import Network.Wai.Handler.Warp qualified as Warp import Network.Wreq qualified as W import OddJobs.Job (Job (..)) +import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter) +import OpenTelemetry.Exporter.OTLP.Span (immutableSpansToProtobuf) import OpenTelemetry.Instrumentation.Hasql qualified as OHasql import OpenTelemetry.Trace (TracerProvider, getGlobalTracerProvider) +import OpenTelemetry.Trace.Core (ImmutableSpan (spanTracer), Tracer (tracerName), createTracerProvider, emptyTracerProviderOptions) import Opentelemetry.OtlpServer qualified as OtlpServer import Pages.BodyWrapper (PageCtx (..)) import Pages.Bots.Slack qualified as SlackBot @@ -650,13 +655,14 @@ runTestBackground t authCtx action = withSharedLogger \logger -> runTestBackgroundWithNotifications :: UTCTime -> Log.Logger -> Config.AuthContext -> ATBackgroundCtx a -> IO ([Data.Effectful.Notify.Notification], a) runTestBackgroundWithNotifications t logger appCtx process = do clock <- newTestClock t - runTestBackgroundWithClock clock logger appCtx process + tp <- getGlobalTracerProvider + runTestBackgroundWithClock tp clock logger appCtx process -- | Like 'runTestBackgroundWithNotifications' but takes an externally-managed -- 'TestClock' so callers can advance time across multiple runs in the same -- spec. -runTestBackgroundWithClock :: TestClock -> Log.Logger -> Config.AuthContext -> ATBackgroundCtx a -> IO ([Data.Effectful.Notify.Notification], a) +runTestBackgroundWithClock :: TracerProvider -> TestClock -> Log.Logger -> Config.AuthContext -> ATBackgroundCtx a -> IO ([Data.Effectful.Notify.Notification], a) runTestBackgroundWithClock = runTestBackgroundWithHTTP (runHTTPGolden "./test/golden/") @@ -666,13 +672,13 @@ runTestBackgroundWithClock = runTestBackgroundWithHTTP (runHTTPGolden "./test/go -- Squeezy with whatever key the config carries. runTestBackgroundWithHTTP :: (forall x hes. IOE :> hes => Eff (HTTP ': hes) x -> Eff hes x) + -> TracerProvider -> TestClock -> Log.Logger -> Config.AuthContext -> ATBackgroundCtx a -> IO ([Data.Effectful.Notify.Notification], a) -runTestBackgroundWithHTTP http clock logger appCtx process = do - tp <- getGlobalTracerProvider +runTestBackgroundWithHTTP http tp clock logger appCtx process = do notifRef <- newIORef [] result <- process @@ -686,7 +692,7 @@ runTestBackgroundWithHTTP http clock logger appCtx process = do & runUUID & runNoSleep & http - & ELLM.runLLMGolden "./test/golden/" + & ELLM.runLLMGolden (ELLM.LLMTelemetry tp ELLM.SpanOnly) "./test/golden/" & CodeReview.runCodeReviewTest (const $ Left "No review agent fixture configured") (const $ Left "No fix agent fixture configured") (const $ Left "No admission agent fixture configured") (const $ Left "No artifact verification fixture configured") (const $ Left "No explanation fixture configured") (const $ Left "No independent fix review fixture configured") & Workspace.runWorkspaceTest (\_ _ _ -> Left "No workspace fixture") (\_ _ -> Left Workspace.RegressionUnavailable) (\_ _ -> Left "No prune fixture") (const $ Left "No maintenance fixture") & Artifacts.runAgentArtifactsTest (const $ Left Artifacts.StorageUnavailable) (\_ _ -> Left Artifacts.ArtifactUnavailable) @@ -706,7 +712,7 @@ runTestBgRecordingHTTP :: UTCTime -> TestResources -> ATBackgroundCtx a -> IO ([ runTestBgRecordingHTTP t TestResources{..} action = do setTestTime trTestClock t reqsRef <- newIORef [] - (notifications, result) <- runTestBackgroundWithHTTP (runHTTPRecord reqsRef) trTestClock trLogger trATCtx action + (notifications, result) <- runTestBackgroundWithHTTP (runHTTPRecord reqsRef) trTracerProvider trTestClock trLogger trATCtx action logNotifications trATCtx trLogger notifications reqs <- reverse <$> readIORef reqsRef pure (reqs, result) @@ -744,7 +750,7 @@ runTestEffect pool hpool logger tp action = do runTestBg :: UTCTime -> TestResources -> ATBackgroundCtx a -> IO a runTestBg t TestResources{..} action = do setTestTime trTestClock t - (notifications, result) <- runTestBackgroundWithClock trTestClock trLogger trATCtx action + (notifications, result) <- runTestBackgroundWithClock trTracerProvider trTestClock trLogger trATCtx action logNotifications trATCtx trLogger notifications pure result @@ -754,14 +760,14 @@ runTestBg t TestResources{..} action = do -- between job runs — otherwise 'runTestBg' would snap the clock back. runTestBgNoReset :: TestResources -> ATBackgroundCtx a -> IO a runTestBgNoReset TestResources{..} action = do - (notifications, result) <- runTestBackgroundWithClock trTestClock trLogger trATCtx action + (notifications, result) <- runTestBackgroundWithClock trTracerProvider trTestClock trLogger trATCtx action logNotifications trATCtx trLogger notifications pure result -- | Capture notifications without resetting the test clock. captureNotifs :: TestResources -> ATBackgroundCtx a -> IO ([Data.Effectful.Notify.Notification], a) -captureNotifs TestResources{..} = runTestBackgroundWithClock trTestClock trLogger trATCtx +captureNotifs TestResources{..} = runTestBackgroundWithClock trTracerProvider trTestClock trLogger trATCtx -- | Log out notifications gathered during a background run. Lifted out @@ -1687,7 +1693,12 @@ ingestLogWithResource tr apiKey attributes bodyText timestamp = -- records never carry span events, so error extraction relies on severity + -- attributes rather than event_attributes.exception. ingestErrorLog :: TestResources -> Text -> Text -> [(Text, Text)] -> UTCTime -> IO () -ingestErrorLog tr apiKey bodyText extras timestamp = +ingestErrorLog tr apiKey = ingestLogRecord tr apiKey ("ERROR", PL.SEVERITY_NUMBER_ERROR) + + +-- | One log record with the given severity, body and string attributes. +ingestLogRecord :: TestResources -> Text -> (Text, PL.SeverityNumber) -> Text -> [(Text, Text)] -> UTCTime -> IO () +ingestLogRecord tr apiKey (severityText, severityNumber) bodyText extras timestamp = let attrs = [mkAttr k v | (k, v) <- extras] logRecord = defMessage @@ -1696,9 +1707,9 @@ ingestErrorLog tr apiKey bodyText extras timestamp = & PLF.body .~ (defMessage & PCF.stringValue .~ bodyText) & PLF.severityText - .~ "ERROR" + .~ severityText & PLF.severityNumber - .~ PL.SEVERITY_NUMBER_ERROR + .~ severityNumber & PLF.attributes .~ attrs scopeLog = defMessage & PLF.logRecords .~ [logRecord] @@ -1765,6 +1776,20 @@ ingestSpanReq :: TestResources -> TS.ExportTraceServiceRequest -> IO () ingestSpanReq tr = void . OtlpServer.traceServiceExport tr.trLogger tr.trATCtx tr.trTracerProvider . Proto +-- | Monoscope observing itself: run @act@ with a tracer that keeps every span it ends, then +-- export those spans through OTLP ingest into the project that @apiKey@ belongs to. +withOwnTelemetry :: TestResources -> Text -> (TestResources -> IO a) -> IO a +withOwnTelemetry tr apiKey act = do + (processor, ended) <- inMemoryListExporter + tp <- createTracerProvider [processor] emptyTracerProviderOptions + result <- act tr{trTracerProvider = tp} + spans <- readIORef ended + exported <- immutableSpansToProtobuf $ HM.fromListWith (<>) [(tracerName (spanTracer imm), V.singleton imm) | imm <- spans] + req <- either (fail . ("re-decoding exported spans: " <>)) pure $ decodeMessage $ encodeMessage exported + ingestSpanReq tr $ req & TSF.resourceSpans . traverse . PTF.resource .~ mkResource apiKey [] + pure result + + withSpanKind :: PT.Span'SpanKind -> TS.ExportTraceServiceRequest -> TS.ExportTraceServiceRequest withSpanKind k = TSF.resourceSpans . traverse . PTF.scopeSpans . traverse . PTF.spans . traverse . PTF.kind .~ k diff --git a/src/System/Config.hs b/src/System/Config.hs index 3da411cac..7ff00977c 100644 --- a/src/System/Config.hs +++ b/src/System/Config.hs @@ -6,6 +6,7 @@ import Data.Base64.Types qualified as B64 import Data.Cache (Cache, newCache) import Data.Char (isHexDigit) import Data.Default (Default (..)) +import Data.Effectful.LLM (GenAIContentCapture) import Data.GeoIP2 qualified as GeoIP2 import Data.Map.Strict qualified as M import Data.Pool as Pool (Pool, defaultPoolConfig, newPool, setNumStripes) @@ -129,6 +130,7 @@ data EnvConfig = EnvConfig , lsWebhookSigEnforce :: Bool , openaiApiKey :: Text , openaiModel :: Text + , otelInstrumentationGenaiCaptureContent :: GenAIContentCapture , impactReviewUsageDir :: Maybe FilePath , impactReviewCacheDir :: Maybe FilePath , impactReviewWorkspaceMaintenance :: Maybe FilePath diff --git a/src/System/Tracing.hs b/src/System/Tracing.hs index da661c5e9..bfab6aa70 100644 --- a/src/System/Tracing.hs +++ b/src/System/Tracing.hs @@ -2,11 +2,13 @@ module System.Tracing ( Tracing, runTracing, withSpan, + withSpanKind, withSpan_, addEvent, addAttribute, setStatus, SpanStatus (..), + SpanKind (..), isExpectedClientErrorText, -- * Cross-thread context propagation @@ -46,7 +48,7 @@ import UnliftIO.Exception (bracket, finally, tryAny, withException) data Tracing :: Effect where - WithSpan :: Text -> [(Text, Attribute)] -> (Span -> m a) -> Tracing m a + WithSpanKind :: SpanKind -> Text -> [(Text, Attribute)] -> (Span -> m a) -> Tracing m a AddEvent :: Span -> Text -> [(Text, Attribute)] -> Tracing m () AddAttribute :: Span -> Text -> Attribute -> Tracing m () SetStatus :: Span -> SpanStatus -> Tracing m () @@ -60,13 +62,13 @@ makeEffect ''Tracing runTracing :: IOE :> es => TracerProvider -> Eff (Tracing ': es) a -> Eff es a runTracing tp = interpret $ \env -> \case - WithSpan name attrs f -> do + WithSpanKind kind name attrs f -> do let tracer = Trace.makeTracer tp "monoscope" Trace.tracerOptions attrMap = HM.fromList attrs -- Properly propagate context through the span lifecycle localSeqUnliftIO env $ \unlift -> liftIO $ do ctx <- Context.getContext - sp <- Trace.createSpan tracer ctx name (defaultSpanArguments{Trace.kind = Server, Trace.attributes = attrMap}) + sp <- Trace.createSpan tracer ctx name (defaultSpanArguments{Trace.kind = kind, Trace.attributes = attrMap}) Context.adjustContext (Context.insertSpan sp) -- Mark the span Error on exception, then always close it + restore -- context (otherwise failed operations show as green spans and we @@ -109,6 +111,10 @@ isExpectedClientErrorText :: Text -> Bool isExpectedClientErrorText = T.isInfixOf "errHTTPCode = 4" +withSpan :: Tracing :> es => Text -> [(Text, Attribute)] -> (Span -> Eff es a) -> Eff es a +withSpan = withSpanKind Server + + withSpan_ :: Tracing :> es => Text -> [(Text, Attribute)] -> Eff es a -> Eff es a withSpan_ name attrs action = withSpan name attrs $ const action diff --git a/src/System/Types.hs b/src/System/Types.hs index c7b0ea6e0..dd89d2d28 100644 --- a/src/System/Types.hs +++ b/src/System/Types.hs @@ -192,7 +192,7 @@ effToServantHandler :: AuthContext -> Log.Logger -> TracerProvider -> ATBaseCtx effToServantHandler env logger tp app = app & Artifacts.runAgentArtifactsReal env - & ELLM.runLLMReal + & ELLM.runLLMReal (ELLM.LLMTelemetry tp env.config.otelInstrumentationGenaiCaptureContent) & Effectful.Reader.Static.runReader env & runUUID & runSleep @@ -224,7 +224,7 @@ effToServantHandlerTestHTTP :: (forall x hes. IOE :> hes => Eff (HTTP ': hes) x effToServantHandlerTestHTTP http clock uuidRef env logger tp app = app & Artifacts.runAgentArtifactsTest (const $ Left Artifacts.StorageUnavailable) (\_ _ -> Left Artifacts.ArtifactUnavailable) - & ELLM.runLLMGolden "./test/golden/" + & ELLM.runLLMGolden (ELLM.LLMTelemetry tp ELLM.SpanOnly) "./test/golden/" & Effectful.Reader.Static.runReader env & runStaticUUIDRef uuidRef & runNoSleep @@ -292,7 +292,7 @@ runBackground logger appCtx tp process = & runUUID & runSleep & runHTTPWreq - & ELLM.runLLMReal + & ELLM.runLLMReal (ELLM.LLMTelemetry tp appCtx.config.otelInstrumentationGenaiCaptureContent) & CodeReview.runCodeReviewReal (CodeReview.ArtifactStore <$> appCtx.config.impactReviewUsageDir <*> appCtx.config.impactReviewCacheDir) (sequenceA $ CodeReview.workerHost (map toString appCtx.config.impactReviewSkills) appCtx.config.impactReviewPrepareScript appCtx.config.impactReviewDependencyArchive appCtx.config.impactReviewWorkspaceRemote <$> appCtx.config.impactReviewGuestImage <*> appCtx.config.impactReviewAgent) appCtx.config.impactReviewAgent (map toString appCtx.config.impactReviewSkills) & Workspace.runWorkspaceReal & Artifacts.runAgentArtifactsReal appCtx diff --git a/src/Utils.hs b/src/Utils.hs index bc6e5a53c..58348f097 100644 --- a/src/Utils.hs +++ b/src/Utils.hs @@ -53,6 +53,7 @@ module Utils ( getSeriesColorHex, nestedJsonFromDotNotation, prettyPrintCount, + tokenFlow, formatWithCommas, formatBytes, showFFloat', @@ -918,6 +919,14 @@ prettyPrintCount n in T.show whole <> memptyIfFalse (tenth /= 0) ("." <> T.show tenth) <> suffix +-- | A model call's input and output tokens, as every LLM surface prints them. +-- +-- >>> tokenFlow (Just 1200) (Just 340) +-- Just "1.2K \8594 340 tok" +tokenFlow :: Maybe Double -> Maybe Double -> Maybe Text +tokenFlow i o = (\a b -> prettyPrintCount (round a) <> " → " <> prettyPrintCount (round b) <> " tok") <$> i <*> o + + -- | Map a raw backend (Postgres / TimeFusion) error message to a short, safe -- user-facing label. Raw detail stays in logs/spans, never the UI. Covers both -- Postgres phrasing and TimeFusion's DataFusion-wrapped forms since prod reads diff --git a/src/Web/Links.hs b/src/Web/Links.hs index f6a411453..76028d71e 100644 --- a/src/Web/Links.hs +++ b/src/Web/Links.hs @@ -275,6 +275,7 @@ data CookieProtectedRoutes mode = CookieProtectedRoutes , dashboardViewDelete :: mode :- "p" :> ProjectId :> "dashboards" :> Capture "dashboard_id" (UUIDId "dashboard") :> "views" :> Capture "name" Text :> AllQueryParams :> Delete '[HTML] (RespHeaders (Opaque "Dashboards.SavedViews")) , dashboardViewsGet :: mode :- "p" :> ProjectId :> "dashboards" :> Capture "dashboard_id" (UUIDId "dashboard") :> "views" :> AllQueryParams :> Get '[HTML] (RespHeaders (Opaque "Dashboards.SavedViews")) , rumRedirect :: mode :- "p" :> ProjectId :> "rum" :> CaptureAll "view" Text :> AllQueryParams :> LocationRedirect NoContent + , llmDashboardRedirect :: mode :- "p" :> ProjectId :> "llm" :> "dashboard" :> AllQueryParams :> LocationRedirect NoContent , dashboardNewGet :: mode :- "p" :> ProjectId :> "dashboards" :> "new" :> Get '[HTML] (RespHeaders (Opaque "Dashboards.DashboardNew")) , dashboardsGet :: mode :- "p" :> ProjectId :> "dashboards" :> Capture "dashboard_id" (UUIDId "dashboard") :> QPT "file" :> QPT "from" :> QPT "to" :> QPT "since" :> HXRequest :> AllQueryParams :> Get '[HTML] (RespHeaders (Opaque1 "PageCtx" (Opaque "Dashboards.DashboardGet"))) , dashboardsGetList :: mode :- "p" :> ProjectId :> "dashboards" :> QPT "sort" :> QPT "embedded" :> QueryParam "teamId" ProjectMembers.TeamId :> QPT "copy_widget_id" :> QueryParam "source_dashboard_id" (UUIDId "dashboard") :> QPT "new" :> RecordParam KeepPrefixExp DashboardFilters :> Get '[HTML] (RespHeaders (Opaque "Dashboards.DashboardsGet")) @@ -514,6 +515,8 @@ data TelemetryRoutes' mode = TelemetryRoutes' , metricCardGetH :: mode :- "metrics" :> "card" :> Capture "metric_name" Text :> QPT "label" :> QPT "metric_source" :> Get '[HTML] (RespHeaders (Opaque "Metrics.MetricFragment")) , serviceMapGetH :: mode :- "service_map" :> QPT "from" :> QPT "to" :> QPT "since" :> QPT "env" :> QPT "endpoint_hash" :> Get '[HTML] (RespHeaders (Opaque "ServiceMap.ServiceMapGet")) , metricServicesGetH :: mode :- "metrics" :> "services" :> QPT "service_search" :> QPT "metric_source" :> Get '[HTML] (RespHeaders (Opaque "Metrics.MetricServices")) + , llmGetH :: mode :- "llm" :> QPT "view" :> QPT "agent" :> QPT "model" :> QPT "signal" :> QPT "from" :> QPT "to" :> QPT "since" :> QPT "deferred" :> Get '[HTML] (RespHeaders (Opaque "LLM.LLMGet")) + , llmConversationGetH :: mode :- "llm" :> "conversations" :> Capture "key" Text :> QPT "from" :> QPT "to" :> QPT "since" :> Get '[HTML] (RespHeaders (Opaque "LLM.ConversationGet")) , hostsGetH :: mode :- "infrastructure" :> "hosts" :> QPT "provider" :> QPT "region" :> QPT "os" :> QPT "integration" :> QPT "group" :> QPT "from" :> QPT "to" :> QPT "since" :> QPT "deferred" :> Get '[HTML] (RespHeaders (Opaque "Infrastructure.HostsGet")) , hostDetailGetH :: mode :- "infrastructure" :> "hosts" :> "detail" :> QPT "host" :> QPT "from" :> QPT "to" :> QPT "since" :> Get '[HTML] (RespHeaders (Opaque "Infrastructure.DetailGet")) , infrastructureContainersGetH :: mode :- "infrastructure" :> "containers" :> QPT "runtime" :> QPT "namespace" :> QPT "node" :> QPT "image" :> QPT "cluster" :> QPT "from" :> QPT "to" :> QPT "since" :> QPT "deferred" :> Get '[HTML] (RespHeaders (Opaque "Containers.ContainersGet")) diff --git a/src/Web/Routes.hs b/src/Web/Routes.hs index d84ade11b..2aea470b6 100644 --- a/src/Web/Routes.hs +++ b/src/Web/Routes.hs @@ -53,6 +53,7 @@ import Pages.Endpoints qualified as ApiCatalog import Pages.GitSync qualified as GitSync import Pages.Infrastructure qualified as Infrastructure import Pages.Issues qualified as IssuesPage +import Pages.LLM qualified as LLM import Pages.LogExplorer.LiveTail qualified as LiveTail import Pages.LogExplorer.Log qualified as Log import Pages.LogExplorer.LogItem qualified as LogItem @@ -339,6 +340,7 @@ cookieProtectedServer projectTools runNow = , slackLinkProjectGet = Slack.linkProjectGetH , endpointDetailsRedirect = Dashboards.entrypointRedirectGetH "endpoint-stats.yaml" , rumRedirect = Dashboards.rumRedirectH + , llmDashboardRedirect = Dashboards.entrypointRedirectGetH "llm-observability.yaml" , dashboardViewPost = Dashboards.dashboardViewPostH , dashboardViewDelete = Dashboards.dashboardViewDeleteH , dashboardViewsGet = Dashboards.dashboardViewsGetH @@ -557,6 +559,8 @@ telemetryServer pid = , metricCardGetH = Metrics.metricCardGetH pid , serviceMapGetH = ServiceMap.serviceMapGetH pid , metricServicesGetH = Metrics.metricServicesGetH pid + , llmGetH = LLM.llmGetH pid + , llmConversationGetH = LLM.conversationGetH pid , hostsGetH = Infrastructure.hostsGetH pid , hostDetailGetH = Infrastructure.hostDetailGetH pid , infrastructureContainersGetH = Containers.containersGetH pid @@ -1086,6 +1090,8 @@ type instance Opaque "GitSync.RepositoryDashboardForm" = GitSync.RepositoryDashb type instance Opaque "GitSync.RepositoryDashboardGet" = GitSync.RepositoryDashboardGet type instance Opaque "GitSync.SelectRejection" = GitSync.SelectRejection type instance Opaque "GitSync.SyncPanels" = GitSync.SyncPanels +type instance Opaque "LLM.LLMGet" = LLM.LLMGet +type instance Opaque "LLM.ConversationGet" = LLM.ConversationGet type instance Opaque "Infrastructure.DetailGet" = Infrastructure.DetailGet type instance Opaque "Infrastructure.HostMapGet" = Infrastructure.HostMapGet type instance Opaque "Infrastructure.HostsGet" = Infrastructure.HostsGet diff --git a/static/model_prices.json b/static/model_prices.json new file mode 100644 index 000000000..69e590dd7 --- /dev/null +++ b/static/model_prices.json @@ -0,0 +1 @@ +{"abliterated-model":[1.0,3.0,0.1,null,262144],"abliterated-model-large":[3.0,5.0,0.3,null,1000000],"abliterated-model-large-flash":[1.0,2.0,0.1,null,1048576],"abliterated-model-large-v2":[3.0,5.0,0.3,null,1000000],"ada":[0.11,0.0,null,null,8191],"agnes-2.5-flash":[0.03,0.15,null,null,512000],"agnes-2.5-pro":[0.45,0.9,0.00378,null,1000000],"ai21.j2-mid-v1":[12.5,12.5,null,null,8191],"ai21.j2-ultra-v1":[18.8,18.8,null,null,8191],"alia-40b-instruct_q8_0":[0.0,0.0,null,null,8192],"allam-1-13b-instruct":[1.8,1.8,null,null,8192],"anthropic-claude-3-opus":[15.0,75.0,null,null,200000],"anthropic-claude-3.5-haiku":[0.8,4.0,null,null,200000],"anthropic-claude-3.5-sonnet":[3.0,15.0,null,null,200000],"anthropic-claude-3.7-sonnet":[3.0,15.0,null,null,200000],"apertus-70b-instruct":[0.0,0.0,null,null,8192],"apertus-8b-instruct":[0.0,0.0,null,null,8192],"aplomb-1":[0.02,0.0,null,null,1000000],"autoglm-phone-9b-multilingual":[0.035,0.138,null,null,65536],"babbage-002":[0.4,0.4,null,null,16384],"baichuan-m2-32b":[0.07,0.07,null,null,131072],"beam":[0.25,1.0,0.05,null,1048576],"beam-1.0":[0.4,1.2,0.04,null,1048576],"beam-1.0-501b-a23b":[0.4,1.2,0.04,null,1048576],"beam-free":[0,0,0,null,1048576],"bge-base-en-v1.5":[0.008,0.0,null,null,512],"bge-en-icl":[0.01,0.0,null,null,32768],"bge-m3":[0.01,0.01,null,null,8192],"bge-multilingual-gemma2":[0.01,0.0,null,null,8192],"c4ai-aya-expanse-32b":[0.5,1.5,null,null,128000],"cc-glm-5.1":[0.06,0.22,null,null,200000],"chat":[0.0,0.0,null,null,null],"chat-latest":[5.0,30.0,0.5,null,400000],"chatdolphin":[0.5,0.5,null,null,16384],"chronos-hermes-13b-v2":[0.2,0.2,null,null,4096],"classify":[0.05,0.0,null,null,null],"claude":[8.0,24.0,null,null,100000],"claude-3-5-haiku":[1.0,5.0,null,null,200000],"claude-3-5-haiku-20241022":[0.8,4.0,0.08,1.0,200000],"claude-3-5-sonnet":[3.0,15.0,0.3,null,200000],"claude-3-5-sonnet-20240620":[3.0,15.0,0.3,3.75,200000],"claude-3-5-sonnet-20241022":[3.0,15.0,0.3,3.75,200000],"claude-3-7-sonnet":[3.0,15.0,0.3,3.75,200000],"claude-3-7-sonnet-20240620":[3.6,18.0,0.36,4.5,200000],"claude-3-7-sonnet-20250219":[3.0,15.0,0.3,3.75,200000],"claude-3-7-sonnet-latest":[3.3,16.5,0.33,null,200000],"claude-3-haiku":[0.25,1.25,0.03,0.3,200000],"claude-3-haiku-20240307":[0.3,1.5,null,null,195904],"claude-3-opus":[15.0,75.0,1.5,18.75,200000],"claude-3-opus-20240229":[15.0,75.0,1.5,18.75,200000],"claude-3-sonnet":[3.0,15.0,null,null,200000],"claude-3.5-haiku":[1.0,5.0,null,null,null],"claude-3.5-sonnet":[3.75,18.75,null,null,null],"claude-3.7-sonnet":[3.0,15.0,null,null,null],"claude-4-opus":[16.5,82.5,null,null,200000],"claude-4-sonnet":[3.3,16.5,null,null,200000],"claude-4.5-haiku":[1.0,5.0,null,null,null],"claude-4.5-sonnet":[3.0,15.0,null,null,null],"claude-fable-5":[10.0,50.0,1.0,12.5,1000000],"claude-fable-5-1":[10.0,50.0,0.25,12.5,1000000],"claude-fable-5-2":[8.0,40.0,0.2,10.0,1000000],"claude-fable-5-2-fast":[16.0,80.0,0.4,20.0,1000000],"claude-fable-5.1":[10.0,50.0,0.25,12.5,200000],"claude-fable-5.2":[8.0,40.0,0.2,10.0,200000],"claude-haiku-4-5":[1.0,5.0,0.1,1.25,200000],"claude-haiku-4-5-20251001":[1.0,5.0,0.1,1.25,200000],"claude-haiku-4.5":[1.0,5.0,0.1,1.25,128000],"claude-haiku-5-5":[0.1,0.5,0.01,0.125,1000000],"claude-haiku-5.5":[0.1,0.5,0.01,0.125,128000],"claude-instant-v1":[0.8,2.4,null,null,100000],"claude-mythos-5":[10.0,50.0,1.0,12.5,1000000],"claude-mythos-5-1":[10.0,50.0,0.25,12.5,1000000],"claude-mythos-5-2":[8.0,40.0,0.2,10.0,1000000],"claude-mythos-5-2-fast":[16.0,80.0,0.4,20.0,1000000],"claude-mythos-preview":[10.0,50.0,1.0,12.5,1000000],"claude-opus-4":[15.0,75.0,null,null,409600],"claude-opus-4-1-20250805":[15.0,75.0,1.5,18.75,200000],"claude-opus-4-20250514":[15.0,75.0,1.5,18.75,200000],"claude-opus-4-5":[5.0,25.0,0.5,6.25,200000],"claude-opus-4-5-20251101":[5.0,25.0,0.5,6.25,200000],"claude-opus-4-6":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-6-20260205":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-6-v1":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-7":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-7-20260416":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-8":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4-8-think":[5.0,25.0,0.5,6.25,1000000],"claude-opus-4.1":[15.0,75.0,1.5,18.75,200000],"claude-opus-4.5":[5.0,25.0,null,null,409600],"claude-opus-4.6":[5.0,25.0,0.5,6.25,200000],"claude-opus-4.7":[5.0,25.0,null,null,null],"claude-opus-4.8":[5.0,25.0,0.5,6.25,200000],"claude-opus-4.8-fast":[10.0,50.0,1.0,12.5,200000],"claude-opus-5":[5.0,25.0,0.5,6.25,1000000],"claude-opus-5-5":[4.0,20.0,0.2,5.0,1000000],"claude-opus-5-5-fast":[8.0,40.0,0.4,10.0,1000000],"claude-opus-5.5":[4.0,20.0,0.2,5.0,200000],"claude-sonnet-4":[3.0,15.0,null,null,409600],"claude-sonnet-4-20250514":[3.0,15.0,0.3,3.75,200000],"claude-sonnet-4-5":[3.0,15.0,0.3,3.75,200000],"claude-sonnet-4-5-20250929":[3.0,15.0,0.3,3.75,200000],"claude-sonnet-4-6":[3.0,15.0,0.3,3.75,1000000],"claude-sonnet-4.5":[3.0,15.0,null,null,409600],"claude-sonnet-4.6":[3.0,15.0,0.3,3.75,200000],"claude-sonnet-5":[2.0,10.0,0.2,2.5,1000000],"claude-sonnet-5-5":[2.0,10.0,0.1,2.5,1000000],"claude-sonnet-5.5":[2.0,10.0,0.1,2.5,200000],"claude-v1":[8.0,24.0,null,null,100000],"cobuddy":[0.28,1.13,0.07,null,131072],"code-llama-13b":[0.2,0.2,null,null,16384],"code-llama-13b-instruct":[0.2,0.2,null,null,16384],"code-llama-13b-python":[0.2,0.2,null,null,16384],"code-llama-34b":[0.9,0.9,null,null,16384],"code-llama-34b-instruct":[0.9,0.9,null,null,16384],"code-llama-34b-python":[0.9,0.9,null,null,16384],"code-llama-70b":[0.9,0.9,null,null,4096],"code-llama-70b-instruct":[0.9,0.9,null,null,4096],"code-llama-70b-python":[0.9,0.9,null,null,4096],"code-llama-7b":[0.2,0.2,null,null,16384],"code-llama-7b-instruct":[0.2,0.2,null,null,16384],"code-llama-7b-python":[0.2,0.2,null,null,16384],"code-qwen-1p5-7b":[0.2,0.2,null,null,65536],"codegeex4":[0.0,0.0,null,null,32768],"codegemma":[0.0,0.0,null,null,8192],"codegemma-2b":[0.1,0.1,null,null,8192],"codegemma-7b":[0.2,0.2,null,null,8192],"codellama":[0.0,0.0,null,null,4096],"codellama-34b-instruct":[0.35,1.4,null,null,16384],"codellama-34b-instruct-hf":[1.0,1.0,null,null,4096],"codellama-70b-instruct":[0.7,2.8,null,null,16384],"codellama-70b-instruct-hf":[1.0,1.0,null,null,4096],"codellama-7b":[0.06,0.12,null,null,16384],"codellama-7b-instruct-awq":[1.923,1.923,null,null,4096],"codestral":[0.3,0.9,null,null,256000],"codestral-2":[0.3,0.9,null,null,128000],"codestral-2405":[0.0,0.0,null,null,32000],"codestral-2501":[0.3,0.9,null,null,256000],"codestral-2508":[0.3,0.9,0.03,null,128000],"codestral-embed":[0.15,0,0.015,null,8192],"codestral-embed-2505":[0.15,0,0.015,null,8192],"codestral-latest":[0.3,0.9,0.03,null,128000],"codestral-mamba-latest":[0.25,0.25,0.025,null,256000],"codex-mini":[1.5,6.0,0.375,null,200000],"coding-glm-5.3":[0.06,0.22,0.015,null,1048576],"coding-kimi-k3":[0.44,1.61333,0.066,null,1048576],"coding-xiaomi-mimo-v2-omni":[0.08,0.4,0.016,null,null],"coding-xiaomi-mimo-v2.5":[0.08,0.16,0.0016,null,1048576],"coding-xiaomi-mimo-v2.5-pro":[0.2,0.4,0.0016,null,1048576],"cogito-671b-v2-p1":[1.2,1.2,null,null,163840],"cogito-v1-preview-llama-3b":[0.1,0.1,null,null,131072],"cogito-v1-preview-llama-70b":[0.9,0.9,null,null,131072],"cogito-v1-preview-llama-8b":[0.2,0.2,null,null,131072],"cogito-v1-preview-qwen-14b":[0.2,0.2,null,null,131072],"cogito-v1-preview-qwen-32b":[0.9,0.9,null,null,131072],"cohere-command-a":[2.5,10.0,null,null,131072],"cohere-command-a-plus-05-2026":[0.8,3.2,null,null,128000],"cohere-embed-v3-english":[0.1,0.0,null,null,512],"cohere-embed-v3-multilingual":[0.1,0.0,null,null,512],"cohere-embed-v5-fast":[0.08,0.0,null,null,128000],"cohere-embed-v5-pro":[0.12,0.0,null,null,128000],"cohere.embed":[0.12,0.0,null,null,128000],"cohere.embed-english-image-v3.0":[0.1,0.0,null,null,512],"cohere.embed-english-light-image-v3.0":[0.1,0.0,null,null,512],"cohere.embed-english-light-v3.0":[0.1,0.0,null,null,512],"cohere.embed-english-v3":[0.1,0.0,null,null,512],"cohere.embed-english-v3.0":[0.1,0.0,null,null,512],"cohere.embed-multilingual-image-v3.0":[0.1,0,null,null,512],"cohere.embed-multilingual-light-image-v3.0":[0.1,0.0,null,null,512],"cohere.embed-multilingual-light-v3.0":[0.1,0.0,null,null,512],"cohere.embed-multilingual-v3":[0.1,0.0,null,null,512],"cohere.embed-multilingual-v3.0":[0.1,0.0,null,null,512],"cohere.embed-v4.0":[0.12,0.0,null,null,128000],"command-a":[2.5,10.0,null,null,256000],"command-a-03-2025":[2.5,10.0,null,null,256000],"command-a-plus-05-2026":[0.0,0.0,null,null,128000],"command-a-reasoning":[1.56,1.56,null,null,256000],"command-a-reasoning-08-2025":[1.56,1.56,null,null,256000],"command-a-translate-08-2025":[0.09,0.09,null,null,256000],"command-a-vision":[1.56,1.56,null,null,256000],"command-a-vision-07-2025":[1.56,1.56,null,null,128000],"command-latest":[1.56,1.56,null,null,128000],"command-light-text-v14":[0.3,0.6,null,null,4096],"command-nightly":[1.0,2.0,null,null,4096],"command-plus-latest":[1.56,1.56,null,null,128000],"command-r":[0.15,0.6,null,null,128000],"command-r-08-2024":[0.15,0.6,null,null,128000],"command-r-plus":[2.5,10.0,null,null,128000],"command-r-plus-08-2024":[2.5,10.0,null,null,128000],"command-r7b-12-2024":[0.0375,0.15,null,null,128000],"command-text-v14":[1.0,2.0,null,null,4096],"community":[0.89,0.89,null,null,64000],"compress":[0.05,0.0,null,null,null],"computer-use-preview":[3.0,12.0,null,null,8192],"computer-use-preview-2025-03-11":[3.0,12.0,null,null,8192],"cosmos3-super-reasoner":[0.1,0.3,null,null,262144],"databricks-bge-large-en":[0.10003,0.0,0.10003,0.10003,512],"databricks-claude-fable-5":[9.99999,50.00002,1.00002,12.49997,1000000],"databricks-claude-fable-5-1":[9.99999,50.00002,0.24997,12.49997,1000000],"databricks-claude-fable-5-2":[8.00002,40.00003,0.19999,9.99999,1000000],"databricks-claude-haiku-4-5":[1.00002,5.00003,0.10003,1.24999,200000],"databricks-claude-haiku-5-5":[0.10003,0.50001,0.01001,0.12502,1000000],"databricks-claude-opus-4":[15.00002,75.0001,1.50003,18.74999,200000],"databricks-claude-opus-4-1":[15.00002,75.0001,1.50003,18.74999,200000],"databricks-claude-opus-4-5":[5.00003,25.00001,0.50001,6.25002,200000],"databricks-claude-opus-4-6":[5.00003,25.00001,0.50001,6.25002,1000000],"databricks-claude-opus-4-7":[5.00003,25.00001,0.50001,6.25002,1000000],"databricks-claude-opus-4-8":[5.00003,25.00001,0.50001,6.25002,1000000],"databricks-claude-opus-5":[5.00003,25.00001,0.50001,6.25002,1000000],"databricks-claude-opus-5-5":[4.00001,19.99998,0.19999,5.00003,1000000],"databricks-claude-sonnet-4":[2.99999,15.00002,0.30002,3.74997,200000],"databricks-claude-sonnet-4-1":[2.99999,15.00002,0.30002,3.74997,200000],"databricks-claude-sonnet-4-5":[2.99999,15.00002,0.30002,3.74997,200000],"databricks-claude-sonnet-4-6":[2.99999,15.00002,0.30002,3.74997,1000000],"databricks-claude-sonnet-5":[1.99997,9.99999,0.19999,2.49998,1000000],"databricks-claude-sonnet-5-5":[1.99997,9.99999,0.10003,2.49998,1000000],"databricks-deepseek-v4-1-flash":[0.30002,1.20001,0.03003,0.30002,1000000],"databricks-deepseek-v4-1-flash-1017":[0.30002,1.20001,0.007,0.30002,1000000],"databricks-deepseek-v4-flash-0731":[0.14,0.28,0.028,0.14,1000000],"databricks-deepseek-v4-pro-0813":[1.31999,3.95997,0.13202,1.31999,1000000],"databricks-gemini-2-5-flash":[0.30002,2.49998,0.030002,0.30002,1048576],"databricks-gemini-2-5-flash-lite":[0.12502,0.50001,0.01253,0.12502,1048576],"databricks-gemini-2-5-pro":[1.24999,9.99999,0.124999,1.24999,1048576],"databricks-gemini-3-1-flash-image":[0.62503,3.74997,0.62503,0.62503,131072],"databricks-gemini-3-1-flash-lite":[0.31248,1.87502,0.03122,0.31248,1048576],"databricks-gemini-3-1-flash-lite-image":[0.31248,1.87502,0.31248,0.31248,65536],"databricks-gemini-3-1-pro":[2.49998,15.00002,0.24997,2.49998,1048576],"databricks-gemini-3-5-flash":[1.87502,11.24998,0.18753,1.87502,1048576],"databricks-gemini-3-5-flash-lite":[0.37499,3.12501,0.03752,0.37499,1048576],"databricks-gemini-3-6-flash":[0.74998,3.74997,0.07497,0.74998,1048576],"databricks-gemini-3-7-flash":[0.74998,3.74997,0.07497,0.74998,1048576],"databricks-gemini-3-8-flash":[0.74998,3.74997,0.07497,0.74998,1048576],"databricks-gemini-3-flash":[0.62503,3.74997,0.06251,0.62503,1048576],"databricks-gemini-3-pro":[2.49998,15.00002,0.24997,2.49998,1048576],"databricks-gemini-3-pro-image":[2.49998,15.00002,2.49998,2.49998,65536],"databricks-gemini-4-argon":[2.49998,12.49997,0.12502,2.49998,1048576],"databricks-gemini-4-carbon":[2.0,10.0,2.0,2.0,1048576],"databricks-gemma-3-12b":[0.15001,0.50001,0.15001,0.15001,128000],"databricks-glm-5-2":[1.4,4.39999,0.25998,1.4,1000000],"databricks-glm-5-3":[1.4,4.39999,0.25998,1.4,1048576],"databricks-glm-5-3-flash":[0.15001,0.50001,0.03003,0.15001,1048576],"databricks-gpt-5":[1.24999,9.99999,0.12502,1.24999,272000],"databricks-gpt-5-1":[1.24999,9.99999,0.12502,1.24999,272000],"databricks-gpt-5-2":[1.75,14.0,0.175,1.75,272000],"databricks-gpt-5-3-codex":[1.75,14.0,0.175,1.75,272000],"databricks-gpt-5-4":[2.49998,15.00002,0.24997,2.49998,922000],"databricks-gpt-5-4-mini":[0.74998,4.50002,0.07497,0.74998,272000],"databricks-gpt-5-4-nano":[0.19999,1.24999,0.02002,0.19999,272000],"databricks-gpt-5-4-pro":[29.99997,180.0001,29.99997,29.99997,922000],"databricks-gpt-5-5":[5.00003,29.99997,0.50001,5.00003,922000],"databricks-gpt-5-5-pro":[29.99997,180.0001,29.99997,29.99997,922000],"databricks-gpt-5-6-luna":[0.19999,1.20001,0.02002,0.24997,922000],"databricks-gpt-5-6-sol":[4.00001,19.99998,0.39998,5.00003,922000],"databricks-gpt-5-6-terra":[1.99997,12.00003,0.19999,2.49998,922000],"databricks-gpt-5-mini":[0.24997,1.99997,0.02499,0.24997,272000],"databricks-gpt-5-nano":[0.04998,0.39998,0.00497,0.04998,272000],"databricks-gpt-6-1-sol":[1.99997,9.99999,0.10003,2.49998,922000],"databricks-gpt-6-1-terra":[1.00002,5.00003,0.04998,1.24999,922000],"databricks-gpt-6-astra":[9.99999,50.00002,1.00002,12.49997,922000],"databricks-gpt-6-luna":[0.10003,0.50001,0.01001,0.12502,922000],"databricks-gpt-6-sol":[1.99997,9.99999,0.19999,2.49998,922000],"databricks-gpt-oss-120b":[0.15001,0.59997,0.15001,0.15001,131072],"databricks-gpt-oss-20b":[0.07,0.30002,0.07,0.07,131072],"databricks-grok-4-6":[2.49998,7.50001,0.62503,2.49998,500000],"databricks-grok-4-7":[2.49998,7.50001,0.62503,2.49998,500000],"databricks-gte-large-en":[0.12999,0.0,0.12999,0.12999,8192],"databricks-inkling":[1.00002,4.04999,0.17003,1.00002,1000000],"databricks-kimi-k2-7":[0.94997,4.00001,0.18998,0.94997,262144],"databricks-kimi-k3":[2.99999,15.00002,0.30002,2.99999,1000000],"databricks-kimi-k3-1-pro":[1.99997,9.99999,0.19999,1.99997,1048576],"databricks-llama-4-maverick":[0.50001,1.50003,0.50001,0.50001,128000],"databricks-meta-llama-3-1-8b-instruct":[0.15001,0.45003,0.15001,0.15001,128000],"databricks-meta-llama-3-3-70b-instruct":[0.50001,1.50003,0.50001,0.50001,128000],"databricks-qwen3-embedding-0-6b":[0.02002,0.0,0.02002,0.02002,32768],"databricks-qwen3-next-80b-a3b-instruct":[0.15001,1.20001,0.15001,0.15001,null],"databricks-qwen35-122b-a10b":[0.22001,2.20003,0.22001,0.22001,262144],"davinci-002":[2.0,2.0,null,null,16384],"daybreak-blue-latest":[4.0,20.0,0.4,5.0,922000],"daybreak-red-latest":[12.5,75.0,1.25,15.625,272000],"dbrx-instruct":[1.2,1.2,null,null,32768],"decisions":[0.05,0.0,null,null,null],"deep-research-max-preview-04-2026":[2.0,12.0,null,null,1048576],"deep-research-preview-04-2026":[2.0,12.0,null,null,1048576],"deep-research-pro-preview-12-2025":[2.0,12.0,null,null,1048576],"deepseek-chat":[0.28,0.42,0.028,null,131072],"deepseek-coder":[0.14,0.28,0.014,null,128000],"deepseek-coder-1b-base":[0.1,0.1,null,null,16384],"deepseek-coder-33b-instruct":[0.9,0.9,null,null,16384],"deepseek-coder-6.7b":[0.06,0.12,null,null,16384],"deepseek-coder-7b-base":[0.2,0.2,null,null,4096],"deepseek-coder-7b-base-v1p5":[0.2,0.2,null,null,4096],"deepseek-coder-7b-instruct-v1p5":[0.2,0.2,null,null,4096],"deepseek-coder-v2-base":[0.0,0.0,null,null,8192],"deepseek-coder-v2-instruct":[1.2,1.2,null,null,65536],"deepseek-coder-v2-lite-base":[0.0,0.0,null,null,8192],"deepseek-coder-v2-lite-instruct":[0.0,0.0,null,null,32768],"deepseek-flash":[0.3,1.2,0.006,0.0,1000000],"deepseek-flash-latest":[0.3,1.2,0.006,null,1048576],"deepseek-llama3.3-70b":[0.2,0.6,null,null,131072],"deepseek-ocr":[0.03,0.03,null,null,8192],"deepseek-ocr-2":[0.03,0.03,null,null,8192],"deepseek-prover-v2":[1.2,1.2,null,null,163840],"deepseek-prover-v2-671b":[0.7,2.5,null,null,160000],"deepseek-r1":[0.55,2.19,0.14,null,65536],"deepseek-r1-0528":[0.5,2.15,0.35,null,163840],"deepseek-r1-0528-distill-qwen3-8b":[0.2,0.2,null,null,131072],"deepseek-r1-0528-maas":[1.35,5.4,null,null,65336],"deepseek-r1-0528-qwen3-8b":[0.06,0.09,null,null,128000],"deepseek-r1-0528-turbo":[1.0,3.0,null,null,32768],"deepseek-r1-671b":[0.8,0.8,null,null,131072],"deepseek-r1-7b-qwen":[0.08,0.15,null,null,131072],"deepseek-r1-8b":[0.1,0.2,null,null,65536],"deepseek-r1-basic":[0.55,2.19,null,null,163840],"deepseek-r1-distill-llama-70b":[0.2,0.6,null,null,131072],"deepseek-r1-distill-llama-8b":[0.025,0.025,null,null,null],"deepseek-r1-distill-qwen-1.5b":[0.09,0.09,null,null,null],"deepseek-r1-distill-qwen-14b":[0.07,0.07,null,null,null],"deepseek-r1-distill-qwen-1p5b":[0.1,0.1,null,null,131072],"deepseek-r1-distill-qwen-32b":[0.497,4.881,null,null,80000],"deepseek-r1-distill-qwen-7b":[0.2,0.2,null,null,null],"deepseek-r1-turbo":[1.0,3.0,null,null,40960],"deepseek-reasoner":[0.28,0.42,0.028,null,131072],"deepseek-v2-lite-chat":[0.5,0.5,null,null,163840],"deepseek-v2p5":[1.2,1.2,null,null,32768],"deepseek-v3":[0.27,1.1,0.07,0.0,65536],"deepseek-v3-0324":[0.24,0.9,0.135,null,163840],"deepseek-v3-2-251201":[0.0,0.0,null,null,98304],"deepseek-v3-turbo":[0.4,1.3,null,null,64000],"deepseek-v3.1":[0.25,0.95,0.13,null,163840],"deepseek-v3.1-maas":[0.6,1.7,0.06,null,163840],"deepseek-v3.1-terminus":[0.27,1.0,0.216,null,163840],"deepseek-v3.2":[0.28,0.4,0.028,null,163840],"deepseek-v3.2-exp":[0.27,0.41,null,null,163840],"deepseek-v3.2-maas":[0.56,1.68,0.056,null,163840],"deepseek-v3.2-speciale":[0.58,1.68,null,null,128000],"deepseek-v3p1":[0.56,1.68,null,null,128000],"deepseek-v3p1-terminus":[0.56,1.68,null,null,128000],"deepseek-v3p2":[0.56,1.68,null,null,163840],"deepseek-v4-1-flash-1017":[0.3,1.2,0.007,null,1048576],"deepseek-v4-flash":[0.3,1.2,0.006,0.0,1000000],"deepseek-v4-flash-0731":[0.44,1.32,0.014,null,1000000],"deepseek-v4-flash-thinking":[0.25,1.75,null,null,200000],"deepseek-v4-flash-vision-exp":[0.3,1.2,0.006,0.0,1000000],"deepseek-v4-pro":[1.32,3.96,0.044,0.0,1000000],"deepseek-v4-pro-0813":[0.92,2.77,0.04,null,1048576],"deepseek-v4-pro-fast":[1.91,3.83,0.16,null,1048576],"deepseek-v4.1-flash":[0.01,1.2,0.0,0.09,1048576],"deepseek-v4.1-flash-1017":[0.3,1.2,0.006,null,1048576],"deepseek-v4.1-flash-1017-fast":[0.6,2.4,0.014,null,1048576],"deepseek-v4.1-flash-260910":[0.3,1.2,0.006,null,1048568],"deepseek-v4.1-flash-abliterated":[0.14,0.5,0.035,null,1048576],"deepseek-v4.1-flash-f2":[0.3,1.2,0.006,null,1048576],"deepseek-v4.1-flash-fast":[0.01,1.2,0.0,0.09,1048576],"deepseek-v4.1-flash-fast-fp4":[0.01,1.2,0.0,0.09,1048576],"deepseek-v4.1-flash-maas":[0.3,1.2,0.03,null,1048576],"deepseek-v4p1-flash":[0.3,1.2,0.006,null,1048576],"deepseek-v4p1-flash-1017":[0.3,1.2,0.006,null,1048576],"deepseek-v4p1-flash-1017-spot":[0.21,0.84,0.0042,null,1048576],"deepseek-v4p1-flash-1017-ultrafast":[2.4,9.6,0.048,null,1048576],"deepseek-v4p1-flash-1017-us":[0.45,1.8,0.009,null,1048576],"deepseek-v4p1-flash-spot":[0.21,0.84,0.0042,null,1048576],"deepseek-v4p1-flash-us":[0.45,1.8,0.009,null,1048576],"deepseek.r1":[1.35,5.4,null,null,128000],"deepseek.v3":[0.58,1.68,null,null,128000],"deepseek.v3.1":[0.58,1.68,null,null,128000],"deepseek.v3.2":[0.62,1.85,null,null,156000],"deepseek_v3":[0.89,0.89,null,null,64000],"devstral-latest":[0.4,2.0,0.04,null,256000],"devstral-medium-latest":[0.4,2.0,0.04,null,256000],"devstral-small":[0.07,0.28,null,null,128000],"devstral-small-2505":[0.9,0.9,null,null,131072],"devstral-small-latest":[0.1,0.3,0.01,null,256000],"dobby-mini-unhinged-plus-llama-3-1-8b":[0.2,0.2,null,null,131072],"dobby-unhinged-llama-3-3-70b-new":[0.9,0.9,null,null,131072],"dolphin":[0.5,0.5,null,null,16384],"dolphin-2-9-2-qwen2-72b":[0.9,0.9,null,null,131072],"dolphin-2p6-mixtral-8x7b":[0.5,0.5,null,null,32768],"dolphin3-8b":[0.08,0.15,null,null,128000],"doubao-embedding":[0.0,0.0,null,null,4096],"doubao-embedding-large":[0.0,0.0,null,null,4096],"doubao-embedding-large-text-240915":[0.0,0.0,null,null,4096],"doubao-embedding-large-text-250515":[0.0,0.0,null,null,4096],"doubao-embedding-text-240715":[0.0,0.0,null,null,4096],"doubao-seed-2-0-code-preview":[0.4822,2.411,0.09644,null,256000],"doubao-seed-2-0-lite-260428":[0.09041,0.54246,0.018082,null,256000],"doubao-seed-2-0-mini":[0.030136,0.30136,0.006027,null,256000],"doubao-seed-2-0-pro":[0.4822,2.411,0.09644,null,256000],"doubao-seed-2-1-lite-260915":[0.115,0.388126,0.023,null,1024000],"doubao-seed-2-1-pro-260628":[0.8625,4.3125,0.1725,null,256000],"doubao-seed-2-1-pro-260915":[0.8625,4.3125,0.1725,null,1024000],"doubao-seed-2-1-turbo":[0.46475,2.32375,0.09295,null,256000],"doubao-seed-2-1-turbo-260628":[0.43125,2.15625,0.08625,null,256000],"e5-mistral-7b-instruct":[0.01,0.0,null,null,32768],"embed-english-light-v3.0":[0.1,0.0,null,null,512],"embed-english-v3.0":[0.1,0.0,null,null,512],"embed-multilingual-light-v3.0":[100.0,0.0,null,null,512],"embed-multilingual-v3.0":[0.1,0.0,null,null,512],"embed-v-4-0":[0.12,0.0,null,null,128000],"embed-v4.0":[0.12,0.0,null,null,128000],"embed-v5.0-fast":[0.08,0.0,null,null,128000],"embed-v5.0-pro":[0.12,0.0,null,null,128000],"embeddings":[0.0,0.0,null,null,512],"embeddings-2":[0.0,0.0,null,null,512],"embeddingsgigar":[0.0,0.0,null,null,4096],"ember-1":[3.0,15.0,0.3,null,1048576],"ernie-4.5-21b-a3b":[0.07,0.28,null,null,120000],"ernie-4.5-21b-a3b-thinking":[0.07,0.28,null,null,131072],"ernie-4.5-300b-a47b-paddle":[0.28,1.1,null,null,123000],"ernie-4.5-vl-28b-a3b":[0.14,0.56,null,null,30000],"ernie-4.5-vl-28b-a3b-thinking":[0.39,0.39,null,null,131072],"ernie-4.5-vl-424b-a47b":[0.42,1.25,null,null,123000],"ernie-4p5-21b-a3b-pt":[0.1,0.1,null,null,131072],"ernie-4p5-300b-a47b-pt":[0.1,0.1,null,null,131072],"ernie-5.1":[0.5634,2.5353,0.5634,null,119000],"eu.deepseek.v3.2":[0.74,2.22,null,null,163840],"extract":[0.05,0.0,null,null,null],"fare-20b":[0.9,0.9,null,null,131072],"firefunction-v1":[0.5,0.5,null,null,32768],"firefunction-v2":[0.9,0.9,null,null,8192],"firellava-13b":[0.2,0.2,null,null,4096],"firesearch-ocr-v6":[0.2,0.2,null,null,8192],"flan-t5-xl-3b":[0.6,0.6,null,null,8192],"flux-1-dev":[0.1,0.1,null,null,4096],"flux-1-dev-controlnet-union":[0.001,0.001,null,null,4096],"flux-1-schnell":[0.1,0.1,null,null,4096],"ft":[3.0,6.0,null,null,16385],"fugu-max":[2.0,6.0,0.25,null,1000000],"fugu-ultra-v1-0":[7.5,45.0,1.5,null,1000000],"fugu-ultra-v1-1":[5.0,30.0,0.5,null,1000000],"fugu-ultra-v2-0":[5.0,30.0,0.5,null,1000000],"fw-deepseek-v3.2":[0.62,1.85,0.31,null,163840],"fw-deepseek-v4-flash":[0.15,0.31,0.03,null,1000000],"fw-deepseek-v4-pro":[1.925,3.828,0.165,null,1000000],"fw-deepseek-v4.1-flash":[0.375,1.5,0.008,null,1000000],"fw-glm-5":[1.1,3.52,0.22,null,200000],"fw-glm-5.1":[1.54,4.84,0.286,null,202800],"fw-glm-5.2":[1.54,4.84,0.15,null,1048576],"fw-glm-5.2-fast":[2.31,7.26,0.231,null,1048576],"fw-glm-5.3":[1.75,5.5,0.325,null,1048576],"fw-glm-5.3-flash":[0.188,0.625,0.038,null,1048576],"fw-gpt-oss-120b":[0.165,0.66,0.082,null,131072],"fw-inkling":[1.1,4.46,0.19,null,1048576],"fw-kimi-k2.5":[0.66,3.3,0.11,null,262144],"fw-kimi-k2.6":[1.045,4.4,0.176,null,262144],"fw-kimi-k2.7-code":[1.05,4.4,0.21,null,262144],"fw-kimi-k3":[3.0,15.0,0.3,null,1048576],"fw-minimax-m2.5":[0.33,1.32,0.033,null,1000000],"fw-minimax-m3":[0.33,1.32,0.066,null,512000],"fw-nemotron-3-ultra-nvfp4":[0.66,2.64,0.13,null,262144],"fw-nemotron-lightning-3.5-30b-a3b":[0.06,0.22,0.01,null,262144],"gemini-2.0-flash":[0.15,0.6,null,null,null],"gemini-2.0-flash-lite":[0.075,0.3,null,null,null],"gemini-2.5-computer-use-preview-10-2025":[1.25,10.0,null,null,128000],"gemini-2.5-flash":[0.3,2.5,0.03,null,1048576],"gemini-2.5-flash-lite":[0.1,0.4,0.01,null,1048576],"gemini-2.5-flash-lite-preview-09-2025":[0.1,0.4,0.01,null,1048576],"gemini-2.5-flash-preview-09-2025":[0.3,2.5,0.03,null,1048576],"gemini-2.5-pro":[1.25,10.0,0.125,null,1048576],"gemini-2.5-pro-preview-tts":[1.0,20.0,0.125,null,8192],"gemini-3-flash":[0.675,4.05,0.0675,null,1048576],"gemini-3-flash-preview":[0.5,3.0,0.05,null,1048576],"gemini-3-flash-preview-search":[0.5,3.0,0.05,null,1048576],"gemini-3-pro":[2.0,12.0,null,null,null],"gemini-3-pro-preview":[2.0,12.0,0.2,null,1048576],"gemini-3.1-flash-lite":[0.25,1.5,0.025,null,1048576],"gemini-3.1-flash-lite-preview":[0.25,1.5,0.025,null,1048576],"gemini-3.1-pro":[2.0,12.0,null,null,1000000],"gemini-3.1-pro-preview":[2.0,12.0,0.2,null,1048576],"gemini-3.1-pro-preview-customtools":[2.0,12.0,0.2,null,1048576],"gemini-3.5-flash":[1.5,9.0,0.15,null,1048576],"gemini-3.5-flash-lite":[0.3,2.5,0.03,null,1048576],"gemini-3.6-flash":[0.75,3.75,0.075,null,1048576],"gemini-3.7-flash":[0.75,3.75,0.075,null,1048576],"gemini-3.8-flash":[0.75,3.75,0.075,null,1048576],"gemini-3.8-flash-cyber":[1.5,7.5,0.15,null,1048576],"gemini-4-argon":[2.0,10.0,0.1,null,1048576],"gemini-4-argon-cyber-permissive":[4.0,20.0,0.2,null,1048576],"gemini-4-carbon":[2.0,10.0,0.1,null,1048576],"gemini-4-carbon-cyber-permissive":[4.0,20.0,0.2,null,1048576],"gemini-4-hydrogen":[1.5,7.5,0.075,null,1048576],"gemini-embedding-001":[0.15,0,null,null,2048],"gemini-embedding-2":[0.2,0,null,null,8192],"gemini-embedding-2-preview":[0.2,0,null,null,8192],"gemini-exp-1114":[0,0,null,null,1048576],"gemini-exp-1206":[0,0,null,null,2097152],"gemini-flash-experimental":[0,0,null,null,1000000],"gemini-flash-latest":[0.75,3.75,0.075,null,1048576],"gemini-flash-lite-latest":[0.3,2.5,0.03,null,1048576],"gemini-gemma-2-27b-it":[0.35,1.05,null,null,8192],"gemini-gemma-2-9b-it":[0.35,1.05,null,null,8192],"gemini-omni-1.1-flash":[1.5,9.0,null,null,131072],"gemini-omni-1.1-flash-preview":[1.5,9.0,null,null,131072],"gemini-omni-1.2-flash":[1.5,9.0,null,null,131072],"gemini-omni-flash-preview":[1.5,9.0,null,null,131072],"gemini-pro-latest":[2.0,12.0,0.2,null,1048576],"gemini-robotics-er-1.5-preview":[0.3,2.5,0,null,1048576],"gemini-robotics-er-2-preview":[1.0,5.0,0.1,null,131072],"gemini-robotics-er-2-streaming-preview":[1.0,5.0,null,null,131072],"gemma-2-9b":[0.2,0.2,null,null,8192],"gemma-2b-it":[0.1,0.1,null,null,8192],"gemma-2b-it-lora":[0.0,0.0,null,null,8192],"gemma-3-12b-it":[0.05,0.15,null,null,131072],"gemma-3-27b-it":[0,0,null,null,131072],"gemma-3-4b-it":[0.05,0.1,null,null,131072],"gemma-3-4b-it-gguf":[0,0,null,null,128000],"gemma-4-12b-it":[0.3,2.0,0.15,null,16384],"gemma-4-26b":[0.03,0.165,null,null,131072],"gemma-4-26b-a4b":[0.05,0.29,0.025,null,262144],"gemma-4-26b-a4b-it":[0.0,0.0,null,null,262144],"gemma-4-26b-a4b-it-maas":[0.15,0.6,0.015,null,262144],"gemma-4-31b":[0.99,1.49,null,null,131072],"gemma-4-31b-it":[0.0,0.0,null,null,262144],"gemma-4-31b-it-nvfp4":[0.14,0.4,0.07,null,262144],"gemma-4-31b-it-thinking":[0.15,0.4,null,null,262144],"gemma-4-31b-it-turbo":[0.09,0.34,0.05,null,262144],"gemma-4-31b-it-ultra":[0.27,0.76,null,null,131072],"gemma-4-e4b-it":[0.02,0.1,null,null,131072],"gemma-7b":[0.2,0.2,null,null,8192],"gemma-7b-it":[0.15,0.15,null,null,8192],"gemma-7b-it-lora":[0.0,0.0,null,null,3500],"gemma-sea-lion-v4-27b-it":[0.351,0.555,null,null,128000],"gemma2-9b-it":[0.2,0.2,null,null,8192],"gemma3-4b":[0.03,0.08,null,null,128000],"gigachat-2":[0.0,0.0,null,null,128000],"gigachat-2-max":[0.0,0.0,null,null,128000],"gigachat-2-pro":[0.0,0.0,null,null,128000],"gigaembeddings-3b-2025-09":[0.0,0.0,null,null,4096],"glimpse-1.0":[0.6,3.0,0.06,null,262144],"glm-4-32b-0414":[0.55,1.66,null,null,32000],"glm-4-32b-0414-128k":[0.1,0.1,null,null,128000],"glm-4-6v-flashx":[0.04,0.4,0.036,null,131072],"glm-4-7-251222":[0.0,0.0,null,null,204800],"glm-4-7-flashx":[0.07,0.4,0.063,null,202752],"glm-4.5":[0.4,1.6,null,null,131072],"glm-4.5-air":[0.2,1.1,null,null,128000],"glm-4.5-air-fp8":[0.2,1.1,null,null,131072],"glm-4.5-airx":[1.1,4.5,null,null,128000],"glm-4.5-flash":[0,0,null,null,128000],"glm-4.5-fp8":[0.6,2.2,null,null,131072],"glm-4.5-x":[2.2,8.9,null,null,128000],"glm-4.5v":[0.6,1.8,null,null,128000],"glm-4.6":[0.6,2.2,null,null,null],"glm-4.6v":[0.3,0.9,0.055,null,131072],"glm-4.7":[0.6,2.2,0.12,null,200000],"glm-4.7-flash":[0.0605,0.4,null,null,131072],"glm-4.7-fp8":[0.4,2.0,null,null,202752],"glm-4.7-h":[0.6,2.2,0.11,null,204800],"glm-4.7-maas":[0.6,2.2,0.06,null,200000],"glm-4p5":[0.55,2.19,null,null,131072],"glm-4p5-air":[0.22,0.88,null,null,131072],"glm-4p5v":[1.2,1.2,null,null,131072],"glm-4p6":[0.55,2.19,null,null,202752],"glm-4p7":[0.6,2.2,0.3,null,202752],"glm-5":[0.95,3.15,null,null,null],"glm-5-1":[0.825,3.301,0.165,null,202745],"glm-5-2":[1.4,4.4,0.14,null,1048576],"glm-5-3":[1.4,4.4,null,null,1048576],"glm-5-3-flash":[0.075,0.25,null,null,1048576],"glm-5-code":[1.2,5.0,0.3,0,200000],"glm-5-fp8":[0.6,1.92,0.12,null,202752],"glm-5-maas":[1.0,3.2,0.1,null,200000],"glm-5-turbo":[1.2,4.0,0.24,null,202800],"glm-5.1":[1.4,4.4,0.26,null,202745],"glm-5.1-fp8":[1.4,4.4,0.26,null,202752],"glm-5.1-nvfp4-mtp":[1.4,4.4,0,null,202752],"glm-5.2":[1.4,4.4,0.26,null,262144],"glm-5.2-fast":[2.1,6.6,0.21,null,1048576],"glm-5.2-fast-preview":[2.254,7.889,0.5635,null,1000000],"glm-5.2-fp8":[1.4,4.4,0.26,null,1048576],"glm-5.2-maas":[1.4,4.4,0.14,null,1000000],"glm-5.3":[1.12,4.4,0.0,1.68,1048576],"glm-5.3-abliterated":[1.0,2.0,0.15,null,1048576],"glm-5.3-fast":[1.12,4.4,0.0,1.68,1048576],"glm-5.3-flash":[0.11,0.35,0.02,null,1048576],"glm-5.3-flash-abliterated":[0.12,0.5,0.04,null,1048576],"glm-5.3-flashx":[0.37,1.25,0.075,0.0,1000000],"glm-5.3-fp4":[1.12,4.4,0.0,1.68,1048576],"glm-5p1":[1.4,4.4,0.26,null,202752],"glm-5p1-fast":[2.8,8.8,0.52,null,202800],"glm-5p2":[1.4,4.4,0.14,null,1048576],"glm-5p2-fast":[2.1,6.6,0.21,null,1048576],"glm-5p2-fast-us":[2.1,6.6,0.21,null,1048576],"glm-5p3":[1.4,4.4,0.26,null,1048576],"glm-5p3-fast":[2.1,6.6,0.39,null,1048576],"glm-5p3-flash":[0.15,0.5,0.03,null,1048576],"glm-5p3-flash-us":[0.225,0.75,0.045,null,1048576],"glm-5p3-spot":[0.98,3.08,0.182,null,1048576],"glm-5p3-us":[2.1,6.6,0.39,null,1048576],"glm-5v-turbo":[1.2,4.0,0.24,null,200000],"glm-fast-latest":[2.1,6.6,0.39,null,1048576],"glm-flash-latest":[0.15,0.5,0.03,null,1048576],"glm-latest":[1.4,4.4,0.26,null,1048576],"global.cohere.embed":[0.12,0.0,null,null,128000],"global.moonshotai.kimi-k3":[3.0,15.0,0.3,3.75,1000000],"global.moonshotai.kimi-k3.1":[4.0,20.0,0.2,5.0,868928],"global.openai.gpt-5.4":[2.5,15.0,0.25,null,922000],"global.openai.gpt-5.5":[5.0,30.0,0.5,null,922000],"global.openai.gpt-5.6-luna":[0.2,1.2,0.02,0.25,922000],"global.openai.gpt-5.6-sol":[4.0,20.0,0.4,5.0,922000],"global.openai.gpt-5.6-terra":[2.0,12.0,0.2,2.5,922000],"global.openai.gpt-6-astra":[10.0,50.0,1.0,12.5,922000],"global.openai.gpt-6-luna":[0.1,0.5,0.01,0.125,922000],"global.openai.gpt-6-sol":[2.0,10.0,0.2,2.5,922000],"global.openai.gpt-6.1-sol":[2.0,10.0,0.1,2.5,1000000],"global.openai.gpt-6.1-terra":[1.0,5.0,0.05,1.25,922000],"global.xai.grok-4.6":[2.0,6.0,0.5,null,500000],"global.xai.grok-4.7":[2.0,6.0,0.5,null,524288],"global.zai.glm-5.3":[1.68,5.28,0.312,2.1,1000000],"google.gemini-2.5-flash":[0.15,0.6,null,null,1048576],"google.gemini-2.5-flash-lite":[0.075,0.3,null,null,1048576],"google.gemini-2.5-pro":[1.25,10.0,null,null,1048576],"google.gemma-3-12b-it":[0.09,0.29,null,null,128000],"google.gemma-3-27b-it":[0.23,0.38,null,null,128000],"google.gemma-3-4b-it":[0.04,0.08,null,null,128000],"google.gemma-4-26b-a4b":[0.13,0.4,null,null,256000],"google.gemma-4-31b":[0.14,0.4,null,null,256000],"google.gemma-4-e2b":[0.04,0.08,null,null,128000],"gpt-3.5-turbo":[0.5,1.5,null,null,16385],"gpt-3.5-turbo-0125":[0.5,1.5,null,null,16385],"gpt-3.5-turbo-1106":[1.0,2.0,null,null,16385],"gpt-3.5-turbo-16k":[3.0,4.0,null,null,16385],"gpt-3.5-turbo-instruct":[1.5,2.0,null,null,8192],"gpt-3.5-turbo-instruct-0914":[1.65,2.2,null,null,4097],"gpt-35-turbo":[0.5,1.5,null,null,4097],"gpt-35-turbo-instruct":[1.65,2.2,null,null,4097],"gpt-35-turbo-instruct-0914":[1.65,2.2,null,null,4097],"gpt-4":[30.0,60.0,null,null,8192],"gpt-4-0125-preview":[10.0,30.0,null,null,128000],"gpt-4-0613":[30.0,60.0,null,null,8192],"gpt-4-1106-preview":[10.0,30.0,null,null,128000],"gpt-4-turbo":[10.0,30.0,null,null,128000],"gpt-4-turbo-2024-04-09":[10.0,30.0,null,null,128000],"gpt-4-turbo-vision-preview":[11.0,33.0,null,null,128000],"gpt-4.1":[2.0,8.0,0.5,null,1047576],"gpt-4.1-2025-04-14":[2.0,8.0,0.5,null,1047576],"gpt-4.1-mini":[0.4,1.6,0.1,null,1047576],"gpt-4.1-mini-2025-04-14":[0.4,1.6,0.1,null,1047576],"gpt-4.1-nano":[0.1,0.4,0.025,null,1047576],"gpt-4.1-nano-2025-04-14":[0.1,0.4,0.025,null,1047576],"gpt-4o":[2.5,10.0,1.25,null,128000],"gpt-4o-2024-05-13":[5.0,15.0,null,null,128000],"gpt-4o-2024-08-06":[2.5,10.0,1.25,null,128000],"gpt-4o-2024-11-20":[2.5,10.0,1.25,null,128000],"gpt-4o-audio-preview-2024-12-17":[2.5,10.0,null,null,128000],"gpt-4o-audio-preview-2025-06-03":[2.5,10.0,null,null,128000],"gpt-4o-mini":[0.15,0.6,0.075,null,128000],"gpt-4o-mini-2024-07-18":[0.15,0.6,0.075,null,128000],"gpt-4o-mini-audio-preview-2024-12-17":[0.15,0.6,null,null,128000],"gpt-4o-mini-search-preview":[0.15,0.6,0.075,null,128000],"gpt-4o-mini-search-preview-2025-03-11":[0.15,0.6,0.075,null,128000],"gpt-4o-search-preview":[2.5,10.0,1.25,null,128000],"gpt-4o-search-preview-2025-03-11":[2.5,10.0,1.25,null,128000],"gpt-5":[1.25,10.0,0.125,null,272000],"gpt-5-2025-08-07":[1.25,10.0,0.125,null,272000],"gpt-5-chat":[1.25,10.0,0.125,null,128000],"gpt-5-chat-latest":[1.25,10.0,0.125,null,128000],"gpt-5-codex":[1.25,10.0,0.125,null,272000],"gpt-5-mini":[0.25,2.0,0.025,null,272000],"gpt-5-mini-2025-08-07":[0.25,2.0,0.025,null,272000],"gpt-5-nano":[0.05,0.4,0.005,null,272000],"gpt-5-nano-2025-08-07":[0.05,0.4,0.005,null,272000],"gpt-5-pro":[15.0,120.0,null,null,400000],"gpt-5-pro-2025-10-06":[15.0,120.0,null,null,400000],"gpt-5-search-api":[1.25,10.0,0.125,null,272000],"gpt-5-search-api-2025-10-14":[1.25,10.0,0.125,null,272000],"gpt-5.1":[1.25,10.0,0.125,null,272000],"gpt-5.1-2025-11-13":[1.25,10.0,0.125,null,272000],"gpt-5.1-chat":[1.25,10.0,0.125,null,111616],"gpt-5.1-chat-latest":[1.25,10.0,0.125,null,128000],"gpt-5.1-codex":[1.25,10.0,0.125,null,400000],"gpt-5.1-codex-2025-11-13":[1.25,10.0,0.125,null,272000],"gpt-5.1-codex-max":[1.25,10.0,0.125,null,400000],"gpt-5.1-codex-mini":[0.25,2.0,0.025,null,400000],"gpt-5.1-codex-mini-2025-11-13":[0.25,2.0,0.025,null,272000],"gpt-5.2":[1.75,14.0,0.175,null,272000],"gpt-5.2-2025-12-11":[1.75,14.0,0.175,null,272000],"gpt-5.2-chat":[1.75,14.0,0.175,null,111616],"gpt-5.2-chat-latest":[1.75,14.0,0.175,null,128000],"gpt-5.2-codex":[1.75,14.0,0.175,null,272000],"gpt-5.2-pro":[21.0,168.0,null,null,272000],"gpt-5.2-pro-2025-12-11":[21.0,168.0,null,null,272000],"gpt-5.3-chat":[1.75,14.0,0.175,null,111616],"gpt-5.3-chat-latest":[1.75,14.0,0.175,null,128000],"gpt-5.3-codex":[1.75,14.0,0.175,null,272000],"gpt-5.4":[2.5,15.0,0.25,null,1050000],"gpt-5.4-2026-03-05":[2.5,15.0,0.25,null,1050000],"gpt-5.4-high":[2.5,15.0,0.25,null,1050000],"gpt-5.4-low":[2.5,15.0,0.25,null,1050000],"gpt-5.4-mini":[0.75,4.5,0.075,null,272000],"gpt-5.4-mini-2026-03-17":[0.75,4.5,0.075,null,272000],"gpt-5.4-nano":[0.2,1.25,0.02,null,272000],"gpt-5.4-nano-2026-03-17":[0.2,1.25,0.02,null,272000],"gpt-5.4-pro":[30.0,180.0,null,null,1050000],"gpt-5.4-pro-2026-03-05":[30.0,180.0,null,null,1050000],"gpt-5.5":[5.0,30.0,0.5,null,1050000],"gpt-5.5-2026-04-23":[5.0,30.0,0.5,null,1050000],"gpt-5.5-2026-04-24":[5.0,30.0,0.5,null,922000],"gpt-5.5-cyber":[12.5,75.0,1.25,null,null],"gpt-5.5-cyber-preview":[12.5,75.0,1.25,null,null],"gpt-5.5-pro":[30.0,180.0,null,null,1050000],"gpt-5.5-pro-2026-04-23":[30.0,180.0,null,null,1050000],"gpt-5.6":[4.0,20.0,0.4,5.0,922000],"gpt-5.6-cyber":[12.5,75.0,1.25,15.625,272000],"gpt-5.6-luna":[0.2,1.2,0.02,0.25,922000],"gpt-5.6-luna-2026-07-09":[0.2,1.2,0.02,0.25,922000],"gpt-5.6-luna-pro":[0.2703,1.621,0.027,null,1050000],"gpt-5.6-sol":[4.0,20.0,0.4,5.0,922000],"gpt-5.6-sol-2026-07-09":[4.0,20.0,0.4,5.0,922000],"gpt-5.6-sol-disc":[4.0,20.0,0.4,5.0,1050000],"gpt-5.6-sol-pro":[5.4,27.0,0.675,null,1050000],"gpt-5.6-terra":[2.0,12.0,0.2,2.5,922000],"gpt-5.6-terra-2026-07-09":[2.0,12.0,0.2,2.5,922000],"gpt-5.6-terra-pro":[2.7,16.2,0.27,null,1050000],"gpt-6-astra":[10.0,50.0,1.0,12.5,922000],"gpt-6-astra-2026-09-03":[10.0,50.0,1.0,12.5,922000],"gpt-6-astra-law":[10.0,50.0,1.0,null,922000],"gpt-6-cyber":[20.0,100.0,2.0,25.0,922000],"gpt-6-luna":[0.1,0.5,0.01,0.125,922000],"gpt-6-luna-2026-09-22":[0.1,0.5,0.01,0.125,922000],"gpt-6-rosalind":[15.0,75.0,1.5,18.75,922000],"gpt-6-sol":[2.0,10.0,0.2,2.5,922000],"gpt-6-sol-2026-09-22":[2.0,10.0,0.2,2.5,922000],"gpt-6.1-sol":[2.0,10.0,0.1,2.5,922000],"gpt-6.1-sol-2026-09-29":[2.0,10.0,0.1,2.5,922000],"gpt-6.1-terra":[1.0,5.0,0.05,1.25,922000],"gpt-audio":[2.5,10.0,null,null,128000],"gpt-audio-1.5":[2.5,10.0,null,null,128000],"gpt-audio-1.5-2026-02-23":[2.5,10.0,null,null,128000],"gpt-audio-2025-08-28":[2.5,10.0,null,null,128000],"gpt-audio-mini":[0.6,2.4,null,null,128000],"gpt-audio-mini-2025-10-06":[0.6,2.4,null,null,128000],"gpt-audio-mini-2025-12-15":[0.6,2.4,null,null,128000],"gpt-chat-latest":[5.0,30.0,0.5,null,128000],"gpt-daybreak-blue-latest":[4.0,20.0,0.4,5.0,922000],"gpt-daybreak-red-latest":[12.5,75.0,1.25,15.625,272000],"gpt-oss":[0.0,0.0,null,null,131072],"gpt-oss-120b":[0.15,0.6,0.075,null,131072],"gpt-oss-120b-maas":[0.09,0.36,null,null,131072],"gpt-oss-120b-mxfp-gguf":[0,0,null,null,131072],"gpt-oss-120b-turbo":[0.15,0.6,null,null,131072],"gpt-oss-120b-ultra":[0.2,0.95,null,null,131072],"gpt-oss-20b":[0.075,0.3,0.0375,null,131072],"gpt-oss-20b-maas":[0.07,0.25,0.007,null,131072],"gpt-oss-20b-mxfp4-gguf":[0,0,null,null,131072],"gpt-oss-safeguard-120b":[1.2,1.2,null,null,131072],"gpt-oss-safeguard-20b":[0.075,0.3,0.037,null,131072],"gpt-rosalind-discovery":[5.0,25.0,0.5,null,null],"gpt-rosalind-research":[5.0,25.0,0.5,null,null],"granite-13b-chat-v2":[0.6,0.6,null,null,8192],"granite-13b-instruct-v2":[0.6,0.6,null,null,8192],"granite-20b-code-instruct-8k":[0.1,0.5,null,null,null],"granite-3-3-8b-instruct":[0.2,0.2,null,null,8192],"granite-3-8b-instruct":[0.2,0.2,null,null,8192],"granite-3.0-2b-instruct":[0.03,0.25,null,null,null],"granite-3.0-8b-instruct":[0.05,0.25,null,null,null],"granite-3.1-2b-instruct":[0.03,0.25,null,null,null],"granite-3.1-8b-instruct":[0.03,0.25,null,null,null],"granite-3.2-8b-instruct":[0.03,0.25,null,null,null],"granite-3.3-8b-instruct":[0.03,0.25,null,null,null],"granite-4-h-small":[0.0636,0.265,null,null,131072],"granite-4.0-h-micro":[0.017,0.112,null,null,131000],"granite-4.0-h-small":[0.06,0.25,null,null,null],"granite-4.1-8b":[0.05,0.1,null,null,131000],"granite-4.2-8b":[0.06,0.15,0.015,null,131000],"granite-8b-code-instruct-128k":[0.05,0.25,null,null,null],"granite-guardian-3-2-2b":[0.1,0.1,null,null,8192],"granite-guardian-3-3-8b":[0.2,0.2,null,null,8192],"granite-ttm-1024-96-r2":[0.38,0.38,null,null,512],"granite-ttm-1536-96-r2":[0.38,0.38,null,null,512],"granite-ttm-512-96-r2":[0.38,0.38,null,null,512],"granite-vision-3-2-2b":[0.1,0.1,null,null,8192],"granite-vision-3.3-2b":[0.0,0.25,null,null,null],"granite-vision-4.1-4b":[0.06,0.25,null,null,null],"grok-2":[2.0,10.0,null,null,131072],"grok-2-vision":[2.0,10.0,null,null,32768],"grok-3":[3.0,15.0,null,null,131072],"grok-3-fast":[5.0,25.0,null,null,131072],"grok-3-mini":[0.3,0.5,null,null,131072],"grok-3-mini-fast":[0.6,4.0,null,null,131072],"grok-4":[3.0,15.0,null,null,262000],"grok-4-1-fast-non-reasoning":[0.2,0.5,null,null,128000],"grok-4-1-fast-reasoning":[0.2,0.5,null,null,128000],"grok-4-20-non-reasoning":[1.25,2.5,1.25,null,262000],"grok-4-20-reasoning":[1.25,2.5,1.25,null,262000],"grok-4-fast-non-reasoning":[0.2,0.5,null,null,null],"grok-4-fast-reasoning":[0.2,0.5,null,null,null],"grok-4.1-fast-non-reasoning":[0.2,0.5,0.05,null,128000],"grok-4.1-fast-reasoning":[0.2,0.5,0.05,null,128000],"grok-4.20":[1.25,2.5,0.2,null,1000000],"grok-4.20-0309":[1.25,2.5,0.2,null,1000000],"grok-4.20-0309-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-0309-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-0309":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-0309-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-0309-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-latest-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-latest-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-beta-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-0304":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-0304-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-0304-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-non-reasoning-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-experimental-beta-reasoning-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-0309":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-beta-0309":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-beta-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-experimental-beta-0304":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-experimental-beta-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-multi-agent-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-non-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-non-reasoning-gv2":[1.25,2.5,0.2,null,1000000],"grok-4.20-non-reasoning-latest":[1.25,2.5,0.2,null,1000000],"grok-4.20-reasoning":[1.25,2.5,0.2,null,1000000],"grok-4.20-reasoning-gv2":[1.25,2.5,0.2,null,1000000],"grok-4.20-reasoning-latest":[1.25,2.5,0.2,null,1000000],"grok-4.3":[1.25,2.5,0.2,null,1000000],"grok-4.3-latest":[1.25,2.5,0.2,null,1000000],"grok-4.5":[2.0,6.0,0.3,null,500000],"grok-4.5-latest":[2.0,6.0,0.3,null,500000],"grok-4.6":[2.0,6.0,0.5,null,500000],"grok-4.7":[2.0,6.0,0.5,null,500000],"grok-build":[1.389,2.777,0.27,null,256000],"grok-build-0.1":[1.0,2.0,0.2,null,256000],"grok-build-latest":[2.0,6.0,0.3,null,500000],"grok-code-fast":[1.0,2.0,0.2,null,256000],"grok-code-fast-1":[1.0,2.0,0.2,null,256000],"grok-code-fast-1-0825":[1.0,2.0,0.2,null,256000],"grok4.6":[2.0,6.0,0.5,null,2000000],"grok4.7":[2.0,6.0,0.5,null,500000],"gte-base":[0.008,0.0,null,null,512],"gte-large":[0.016,0.0,null,null,512],"hermes-2-pro-llama-3-8b":[0.14,0.14,null,null,8192],"hermes-2-pro-mistral-7b":[0.2,0.2,null,null,32768],"hermes-3-8b-tee":[0.15,0.6,null,null,16000],"hermes-3-llama-3.1-405b":[1.0,1.0,null,null,131072],"hermes-3-llama-3.1-70b":[0.7,0.7,null,null,131072],"hermes-4-405b":[1.0,3.0,0.2,null,131072],"hermes-4-70b":[0.13,0.4,null,null,131072],"hermes3-405b":[0.8,0.8,null,null,131072],"hermes3-70b":[0.12,0.3,null,null,131072],"hermes3-8b":[0.025,0.04,null,null,131072],"hunyuanocr":[0.5,0.5,null,null,32768],"hy3":[0.14,0.58,0.035,null,262144],"hy3-preview":[0.18,0.6,0.06,null,262144],"hy4":[0.417,1.2505,0.021,null,960000],"hy4-preview":[0.417,1.2505,0.021,null,960000],"in.moonshotai.kimi-k3":[3.3,16.5,0.33,4.125,1000000],"in.moonshotai.kimi-k3.1":[4.4,22.0,0.22,5.5,868928],"inkling":[1.0,4.05,0.17,null,524288],"inkling-2-small":[0.3,1.0,0.03,null,1048576],"inkling-small":[0.45,1.2,0.1,null,524288],"internlm2_5-20b-chat":[0.0,0.0,null,null,32768],"internvl3-38b":[0.9,0.9,null,null,16384],"internvl3-78b":[0.9,0.9,null,null,16384],"internvl3-8b":[0.2,0.2,null,null,16384],"isaac-0.1":[0.26,2.17,null,null,null],"j2-light":[3.0,3.0,null,null,8192],"j2-mid":[10.0,10.0,null,null,8192],"j2-ultra":[15.0,15.0,null,null,8192],"jais-13b-chat":[1.8,1.8,null,null,8192],"jais-30b-chat":[3.2,9.71,null,null,8192],"jamba-1-5-large":[2.0,8.0,null,null,256000],"jamba-1-5-mini":[0.2,0.4,null,null,256000],"jamba-1.5":[0.2,0.4,null,null,256000],"jamba-1.5-large":[2.0,8.0,null,null,256000],"jamba-1.5-mini":[0.2,0.4,null,null,256000],"jamba-instruct":[0.5,0.7,null,null,70000],"jamba-large-1.6":[2.0,8.0,null,null,256000],"jamba-large-1.7":[2.0,8.0,null,null,256000],"jamba-mini-1.6":[0.2,0.4,null,null,256000],"jamba-mini-1.7":[0.2,0.4,null,null,256000],"kat-coder":[0.9,0.9,null,null,262144],"kat-coder-pro":[0.3,1.2,0.06,null,256000],"kat-dev-32b":[0.9,0.9,null,null,131072],"kat-dev-72b-exp":[0.9,0.9,null,null,131072],"kimi-fast-latest":[4.5,22.5,0.45,null,1048576],"kimi-k2":[0.55,2.2,null,null,131072],"kimi-k2-0905":[0.6,2.5,null,null,262144],"kimi-k2-6":[0.8939,3.7131,0.1788,null,262144],"kimi-k2-7-code":[0.95,4.0,null,null,262144],"kimi-k2-7-code-highspeed":[1.9,8.0,null,null,262144],"kimi-k2-instruct":[0.5,2.0,null,null,131072],"kimi-k2-instruct-0905":[0.5,2.0,0.4,null,262144],"kimi-k2-thinking":[0.6,2.5,null,null,262144],"kimi-k2-thinking-251104":[0.0,0.0,null,null,229376],"kimi-k2-thinking-maas":[0.6,2.5,0.06,null,256000],"kimi-k2.5":[0.6,3.0,0.1,null,262144],"kimi-k2.5-fp4":[0.5,2.8,null,null,262144],"kimi-k2.6":[0.95,4.0,0.16,null,262144],"kimi-k2.7-code":[0.95,4.0,0.19,null,262144],"kimi-k2.7-code-highspeed":[1.9,8.0,0.38,0.0,256000],"kimi-k2.8-preview":[1.0,4.0,0.25,0.0,1000000],"kimi-k2p5":[0.6,3.0,0.1,null,262144],"kimi-k2p6":[0.95,4.0,0.16,null,262144],"kimi-k2p6-fast":[2.0,8.0,0.3,null,262144],"kimi-k2p7-code":[0.95,4.0,0.19,null,262144],"kimi-k2p7-code-fast":[1.9,8.0,0.38,null,262144],"kimi-k3":[2.5,12.5,0.25,null,1048576],"kimi-k3-1-pro":[2.0,10.0,1.8,null,1048576],"kimi-k3-fast":[4.5,22.5,0.45,null,1048576],"kimi-k3-spot":[2.1,10.5,0.21,null,1048576],"kimi-k3-us":[4.5,22.5,0.45,null,1048576],"kimi-k3.1-pro":[2.0,10.0,0.2,2.0,1048576],"kimi-k3.1-pro-highspeed":[4.0,20.0,0.2,4.0,1048576],"kimi-k3p1-pro":[2.0,10.0,0.2,null,1048576],"kimi-latest":[3.0,15.0,0.3,null,1048576],"l3-70b-euryale-v2.1":[1.48,1.48,null,null,8192],"l3-8b-lunaris":[0.05,0.05,null,null,8192],"l3-8b-lunaris-v1-turbo":[0.04,0.05,null,null,8192],"l3-8b-stheno-v3.2":[0.05,0.05,null,null,8192],"l3.1-70b-euryale-v2.2":[0.85,0.85,null,null,131072],"l3.3-70b-euryale-v2.3":[0.65,0.75,null,null,131072],"l31-70b-euryale-v2.2":[1.48,1.48,null,null,8192],"labs-leanstral-1-5":[0.0,0.0,null,null,262144],"labs-leanstral-1-5-1":[0.0,0.0,null,null,262144],"learnlm-1.5-pro-experimental":[0,0,null,null,32767],"lfm-40b":[0.1,0.2,null,null,131072],"lfm-7b":[0.025,0.04,null,null,131072],"lfm2.5-1.2b-instruct":[0.02,0.05,null,null,32768],"lfm2.5-1.2b-thinking":[0.02,0.05,null,null,32768],"ling-3.0-flash":[0.06,0.18,0.012,null,262144],"ling-3.0-flash-fast":[0.06,0.18,0.012,null,262144],"llama-2-13b":[0.1,0.5,null,null,4096],"llama-2-13b-chat":[0.1,0.5,null,null,4096],"llama-2-13b-chat-hf":[0.25,0.25,null,null,4096],"llama-2-70b":[0.65,2.75,null,null,4096],"llama-2-70b-chat":[0.7,2.8,null,null,4096],"llama-2-70b-chat-hf":[1.0,1.0,null,null,4096],"llama-2-7b":[0.05,0.25,null,null,4096],"llama-2-7b-chat":[0.05,0.25,null,null,4096],"llama-2-7b-chat-fp16":[1.923,1.923,null,null,3072],"llama-2-7b-chat-hf":[0.15,0.15,null,null,4096],"llama-2-7b-chat-hf-lora":[0.0,0.0,null,null,8192],"llama-2-7b-chat-int8":[1.923,1.923,null,null,2048],"llama-3-2-11b-vision-instruct":[0.35,0.35,null,null,128000],"llama-3-2-1b-instruct":[0.1,0.1,null,null,128000],"llama-3-2-3b-instruct":[0.15,0.15,null,null,128000],"llama-3-2-90b-vision-instruct":[2.0,2.0,null,null,128000],"llama-3-3-70b-instruct":[0.7526,0.7526,null,null,131072],"llama-3-70b":[0.65,2.75,null,null,8192],"llama-3-70b-instruct":[0.65,2.75,null,null,8192],"llama-3-8b":[0.05,0.25,null,null,8086],"llama-3-8b-instruct":[0.05,0.25,null,null,8086],"llama-3.1-405b-instruct":[10.68,10.68,null,null,128000],"llama-3.1-405b-instruct-maas":[5.0,16.0,null,null,128000],"llama-3.1-70b":[0.6,0.6,null,null,128000],"llama-3.1-70b-instruct":[0.72,0.72,null,null,128000],"llama-3.1-70b-instruct-maas":[0.0,0.0,null,null,128000],"llama-3.1-8b":[0.1,0.1,null,null,128000],"llama-3.1-8b-instant":[0.2,0.5,null,null,131072],"llama-3.1-8b-instruct":[0.03,0.03,null,null,null],"llama-3.1-8b-instruct-fast":[0.15,0.28,0.025,null,131072],"llama-3.1-8b-instruct-fp8":[0.152,0.287,null,null,32000],"llama-3.1-8b-instruct-maas":[0.0,0.0,null,null,128000],"llama-3.1-nemotron-70b-instruct":[0.6,0.6,null,null,131072],"llama-3.1-nemotron-ultra-253b-v1":[0.6,1.8,null,null,128000],"llama-3.2-11b":[0.16,0.16,null,null,128000],"llama-3.2-11b-vision-instruct":[0.0485,0.676,null,null,128000],"llama-3.2-1b":[0.1,0.1,null,null,128000],"llama-3.2-1b-instruct":[0.027,0.201,null,null,60000],"llama-3.2-3b":[0.15,0.15,null,null,128000],"llama-3.2-3b-instruct":[0.0509,0.335,null,null,80000],"llama-3.2-3b-instruct-fast":[0.05,0.3,0.025,null,131072],"llama-3.2-90b":[0.72,0.72,null,null,128000],"llama-3.2-90b-vision-instruct":[2.0,2.0,null,null,128000],"llama-3.2-90b-vision-instruct-maas":[0.0,0.0,null,null,128000],"llama-3.3-70b":[0.85,1.2,null,null,128000],"llama-3.3-70b-instruct":[0.71,0.71,null,null,128000],"llama-3.3-70b-instruct-fp8-dynamic":[0.72,0.72,null,null,128000],"llama-3.3-70b-instruct-fp8-fast":[0.293,2.253,null,null,24000],"llama-3.3-70b-instruct-maas":[0.72,0.72,null,null,128000],"llama-3.3-70b-instruct-turbo":[0.1,0.32,null,null,131072],"llama-3.3-70b-versatile":[0.2,0.5,null,null,131072],"llama-3.3-nemotron-super-49b-v1":[0.1,0.4,null,null,131072],"llama-3.3-nemotron-super-49b-v1.5":[0.1,0.4,null,null,131072],"llama-3_1-nemotron-ultra-253b-v1":[0.6,1.8,null,null,131072],"llama-4-maverick":[0.2,0.6,null,null,131072],"llama-4-maverick-17b":[0.371,1.484,null,null,131072],"llama-4-maverick-17b-128e-instruct":[0.63,1.8,null,null,131072],"llama-4-maverick-17b-128e-instruct-fp8":[0.25,1.0,null,null,1000000],"llama-4-maverick-17b-128e-instruct-maas":[0.35,1.15,null,null,1000000],"llama-4-maverick-17b-16e-instruct-maas":[0.35,1.15,null,null,1000000],"llama-4-maverick-instruct":[0.25,0.95,null,null,null],"llama-4-scout":[0.1,0.3,null,null,131072],"llama-4-scout-17b-128e-instruct-maas":[0.25,0.7,null,null,10000000],"llama-4-scout-17b-16e-instruct":[0.2,0.78,null,null,10000000],"llama-4-scout-17b-16e-instruct-maas":[0.25,0.7,null,null,10000000],"llama-4-scout-instruct":[0.17,0.65,null,null,null],"llama-guard-2-8b":[0.2,0.2,null,null,8192],"llama-guard-3-11b-vision":[0.35,0.35,null,null,128000],"llama-guard-3-1b":[0.1,0.1,null,null,131072],"llama-guard-3-8b":[0.484,0.03,null,null,131072],"llama-guard-4-12b":[0.18,0.18,null,null,163840],"llama-prompt-guard-2-22m":[0.03,0.03,null,null,512],"llama-prompt-guard-2-86m":[0.04,0.04,null,null,512],"llama-v2-13b":[0.2,0.2,null,null,4096],"llama-v2-13b-chat":[0.2,0.2,null,null,4096],"llama-v2-70b":[0.1,0.1,null,null,4096],"llama-v2-70b-chat":[0.9,0.9,null,null,2048],"llama-v2-7b":[0.2,0.2,null,null,4096],"llama-v2-7b-chat":[0.2,0.2,null,null,4096],"llama-v3-70b-instruct":[0.9,0.9,null,null,8192],"llama-v3-70b-instruct-hf":[0.9,0.9,null,null,8192],"llama-v3-8b":[0.2,0.2,null,null,8192],"llama-v3-8b-instruct-hf":[0.2,0.2,null,null,8192],"llama-v3p1-405b-instruct":[3.0,3.0,null,null,131072],"llama-v3p1-405b-instruct-long":[0.1,0.1,null,null,4096],"llama-v3p1-70b-instruct":[0.9,0.9,null,null,131072],"llama-v3p1-70b-instruct-1b":[0.1,0.1,null,null,4096],"llama-v3p1-8b-instruct":[0.1,0.1,null,null,131072],"llama-v3p1-nemotron-70b-instruct":[0.9,0.9,null,null,131072],"llama-v3p2-11b-vision-instruct":[0.2,0.2,null,null,131072],"llama-v3p2-1b":[0.1,0.1,null,null,131072],"llama-v3p2-1b-instruct":[0.1,0.1,null,null,131072],"llama-v3p2-3b":[0.1,0.1,null,null,131072],"llama-v3p2-3b-instruct":[0.1,0.1,null,null,131072],"llama-v3p2-90b-vision-instruct":[0.9,0.9,null,null,131072],"llama-v3p3-70b-instruct":[0.9,0.9,null,null,131072],"llama2":[0.0,0.0,null,null,4096],"llama2-13b-chat-v1":[0.75,1.0,null,null,4096],"llama2-70b-chat-v1":[1.95,2.56,null,null,4096],"llama2-uncensored":[0.0,0.0,null,null,4096],"llama3":[0.0,0.0,null,null,8192],"llama3-1-405b-instruct":[2.4,2.4,null,null,128000],"llama3-1-70b-instruct":[0.72,0.72,null,null,128000],"llama3-1-8b-instruct":[0.22,0.22,null,null,128000],"llama3-2-11b-instruct":[0.16,0.16,null,null,128000],"llama3-2-1b-instruct":[0.13,0.13,null,null,128000],"llama3-2-3b-instruct":[0.19,0.19,null,null,128000],"llama3-2-90b-instruct":[0.72,0.72,null,null,128000],"llama3-3-70b-instruct":[0.72,0.72,null,null,128000],"llama3-405b-instruct-maas":[0.0,0.0,null,null,32000],"llama3-70b-instruct":[3.18,4.2,null,null,8192],"llama3-70b-instruct-maas":[0.0,0.0,null,null,32000],"llama3-8b-instruct":[0.36,0.72,null,null,8192],"llama3-8b-instruct-maas":[0.0,0.0,null,null,32000],"llama3.1":[0.0,0.0,null,null,8192],"llama3.1-405b":[1.2,1.2,null,null,128000],"llama3.1-405b-instruct-fp8":[0.8,0.8,null,null,131072],"llama3.1-70b":[0.72,0.72,null,null,128000],"llama3.1-70b-instruct-fp8":[0.12,0.3,null,null,131072],"llama3.1-8b":[0.1,0.1,null,null,128000],"llama3.1-8b-instruct":[0.025,0.04,null,null,131072],"llama3.1-nemotron-70b-instruct-fp8":[0.12,0.3,null,null,131072],"llama3.2-11b-vision-instruct":[0.015,0.025,null,null,131072],"llama3.2-3b-instruct":[0.015,0.025,null,null,131072],"llama3.3-70b":[0.72,0.72,null,null,128000],"llama3.3-70b-instruct":[0.65,0.65,null,null,128000],"llama3.3-70b-instruct-fp8":[0.12,0.3,null,null,131072],"llama4-maverick":[0.24,0.97,null,null,128000],"llama4-maverick-17b-instruct":[0.24,0.97,null,null,1000000],"llama4-maverick-instruct-basic":[0.22,0.88,null,null,131072],"llama4-scout-17b-instruct":[0.17,0.66,null,null,10000000],"llama4-scout-instruct-basic":[0.15,0.6,null,null,131072],"llamaguard-7b":[0.2,0.2,null,null,4096],"llava-7b":[0.1,0.2,null,null,4096],"llava-v1.6-mistral-7b-hf":[0.29,0.29,null,null,32000],"llava-yi-34b":[0.9,0.9,null,null,4096],"longcat-2.0":[0.7746,3.0984,0.015492,null,1000000],"longcat-2.5-preview-free":[0,0,0,null,1000000],"lyria-3-clip-preview":[0,0,null,null,131072],"lyria-3-pro-preview":[0,0,null,null,131072],"lyria-3.5":[0,0,null,null,131072],"lyria-3.5-clip-preview":[0,0,null,null,131072],"lyria-3.5-pro-preview":[0,0,null,null,131072],"lyria-realtime-exp":[0,0,null,null,1048576],"macaron-v1-tall":[0.2475,1.43,0.044,null,262144],"macaron-v1-venti":[0.825,2.475,0.165,null,1048576],"magistral-medium":[2.0,5.0,null,null,128000],"magistral-medium-latest":[1.5,7.5,0.15,null,262144],"magistral-small":[0.5,1.5,null,null,128000],"magistral-small-latest":[0.15,0.6,0.015,null,262144],"mai-code-1.1-flash":[0.2,1.2,0.02,null,128000],"mai-cyber-1-flash":[0.6,3.5,0.06,null,256000],"mai-ds-r1":[1.35,5.4,null,null,128000],"mai-thinking-1":[2.0,8.0,0.2,null,256000],"mamba-codestral-7b-v0.1":[0.19,0.19,null,null,256000],"mellum2-12b-a2.5b-instruct":[0.05,0.1,null,null,131000],"mercury-2":[0.25,0.75,0.025,null,128000],"mercury-2.5":[0.2,0.75,0.02,null,260000],"mercury-coder-small":[0.25,1.0,null,null,32000],"mercury-edit-2":[0.25,0.75,0.025,null,32000],"meta-llama-3-70b":[0.65,2.75,null,null,null],"meta-llama-3-70b-instruct":[1.0,1.0,null,null,8192],"meta-llama-3-8b":[0.05,0.25,null,null,null],"meta-llama-3-8b-instruct":[0.15,0.15,null,null,8192],"meta-llama-3.1-405b-instruct":[0.12,0.3,null,null,32768],"meta-llama-3.1-70b-instruct":[0.4,0.4,null,null,131072],"meta-llama-3.1-70b-instruct-turbo":[0.4,0.4,null,null,131072],"meta-llama-3.1-8b":[0.2,0.2,null,null,16384],"meta-llama-3.1-8b-instruct":[0.03,0.05,null,null,131072],"meta-llama-3.1-8b-instruct-turbo":[0.02,0.04,null,null,131072],"meta-llama-3.3-70b-instruct":[0.6,1.2,null,null,131072],"meta-llama-3_1-70b-instruct":[0.67,0.67,null,null,131000],"meta-llama-3_3-70b-instruct":[0.67,0.67,null,null,131000],"meta-textgeneration-llama-2-13b":[0.0,0.0,null,null,4096],"meta-textgeneration-llama-2-13b-f":[0.0,0.0,null,null,4096],"meta-textgeneration-llama-2-70b":[0.0,0.0,null,null,4096],"meta-textgeneration-llama-2-70b-b-f":[0.0,0.0,null,null,4096],"meta-textgeneration-llama-2-7b":[0.0,0.0,null,null,4096],"meta-textgeneration-llama-2-7b-f":[0.0,0.0,null,null,4096],"mimo-v2-6-flash":[0.14,0.28,null,null,1048576],"mimo-v2-6-pro":[0.435,0.87,null,null,1048576],"mimo-v2-6-pro-ultraspeed":[4.35,8.7,null,null,1048576],"mimo-v2-flash":[0.11,0.33,0.024,null,262144],"mimo-v2-omni":[0.44,2.2,0.088,null,256000],"mimo-v2-pro":[1.1,3.3,0.22,null,1000000],"mimo-v2.5":[0.168,0.336,0.0034,null,1048576],"mimo-v2.5-pro":[0.522,1.044,0.0043,null,1048576],"mimo-v2.6-flash":[0.14,0.28,0.0028,null,null],"mimo-v2.6-pro":[0.435,0.87,0.0036,null,null],"minicpm-v-4_5":[0.658,1.11,0.13,null,32000],"minimax-latest":[0.3,1.2,0.06,null,512000],"minimax-m1-80k":[0.1,0.1,null,null,4096],"minimax-m2":[0.3,1.2,0.03,0.375,200000],"minimax-m2-7":[0.15,0.6,0.03,null,204800],"minimax-m2-7-highspeed":[0.3,1.2,0.03,null,204800],"minimax-m2-maas":[0.3,1.2,0.03,null,196608],"minimax-m2.1":[0.3,1.2,null,null,196608],"minimax-m2.1-lightning":[0.3,2.4,0.03,0.375,1000000],"minimax-m2.5":[0.3,1.2,null,null,null],"minimax-m2.5-highspeed":[0.6,2.4,0.03,null,204800],"minimax-m2.5-lightning":[0.3,2.4,0.03,0.375,1000000],"minimax-m2.7":[0.6,2.4,null,null,196608],"minimax-m2.7-highspeed":[0.6,2.4,0.06,null,204800],"minimax-m2.7-turbo":[0.38,1.7,0.07,null,196608],"minimax-m2p1":[0.3,1.2,0.03,null,196608],"minimax-m3":[0.3,1.2,0.06,null,512000],"minimax-m3-1-flash":[0.1125,0.45,0.01125,null,1048576],"minimax-m3.1-flash":[0.15,0.6,0.015,null,1000000],"minimax-m3p1-flash":[0.15,0.6,0.015,null,1048576],"minimax.minimax-m2":[0.3,1.2,null,null,1000000],"minimax.minimax-m2.1":[0.3,1.2,null,null,196000],"minimax.minimax-m2.5":[0.3,1.2,null,null,196000],"ministral-14b-2512":[0.2,0.2,0.02,null,262144],"ministral-14b-latest":[0.2,0.2,0.02,null,262144],"ministral-3-14b-2512":[0.2,0.2,0.02,null,262144],"ministral-3-14b-instruct-2512":[0.2,0.2,null,null,256000],"ministral-3-3b-2512":[0.1,0.1,0.01,null,131072],"ministral-3-3b-instruct-2512":[0.1,0.1,null,null,256000],"ministral-3-8b-2512":[0.15,0.15,0.015,null,262144],"ministral-3-8b-instruct-2512":[0.2,0.2,null,null,256000],"ministral-3b":[0.04,0.04,null,null,128000],"ministral-3b-2512":[0.1,0.1,0.01,null,131072],"ministral-3b-latest":[0.1,0.1,0.01,null,131072],"ministral-8b":[0.1,0.1,null,null,128000],"ministral-8b-2512":[0.15,0.15,0.015,null,262144],"ministral-8b-latest":[0.15,0.15,0.015,null,262144],"mistral":[0.0,0.0,null,null,8192],"mistral-7b":[0.2,0.2,null,null,32768],"mistral-7b-instruct":[0.2,0.26,null,null,32000],"mistral-7b-instruct-4k":[0.2,0.2,null,null,32768],"mistral-7b-instruct-v0.1":[0.15,0.15,null,null,16384],"mistral-7b-instruct-v0.2":[0.0,0.0,null,null,32768],"mistral-7b-instruct-v0.2-lora":[0.0,0.0,null,null,15000],"mistral-7b-instruct-v0.3":[0.1,0.1,null,null,127000],"mistral-7b-instruct-v0p2":[0.2,0.2,null,null,32768],"mistral-7b-instruct-v3":[0.2,0.2,null,null,32768],"mistral-7b-v0.1":[0.05,0.25,null,null,4096],"mistral-7b-v0.3":[0.1,0.15,null,null,32768],"mistral-7b-v0p2":[0.2,0.2,null,null,32768],"mistral-code-agent-latest":[0.4,2.0,0.04,null,256000],"mistral-code-fim-latest":[0.3,0.9,0.03,null,128000],"mistral-code-latest":[0.3,0.9,0.03,null,128000],"mistral-embed":[0.1,0,0.01,null,8192],"mistral-embed-2312":[0.1,0,0.01,null,8192],"mistral-large":[2.0,6.0,null,null,32000],"mistral-large-2402":[5.2,15.6,null,null,32000],"mistral-large-2407":[2.0,6.0,null,null,128000],"mistral-large-2411":[2.0,6.0,null,null,128000],"mistral-large-2512":[0.5,1.5,0.05,null,262144],"mistral-large-3":[0.5,1.5,0.05,null,262144],"mistral-large-3-675b-instruct":[0.5,1.5,null,null,256000],"mistral-large-3-fp8":[1.2,1.2,null,null,256000],"mistral-large-4":[0.7,2.1,0.07,null,1048576],"mistral-large-4-0":[0.7,2.1,0.07,null,1048576],"mistral-large-4-latest":[0.7,2.1,null,null,1048576],"mistral-large-instruct-2407":[0.0,0.0,null,null,65536],"mistral-large-latest":[0.5,1.5,0.05,null,262144],"mistral-large2":[2.0,6.0,null,null,128000],"mistral-medium":[1.5,7.5,0.15,null,262144],"mistral-medium-2505":[0.4,2.0,null,null,131072],"mistral-medium-2604":[1.5,7.5,0.15,null,262144],"mistral-medium-3":[1.5,7.5,0.15,null,262144],"mistral-medium-3-5":[1.5,7.5,0.15,null,262144],"mistral-medium-3.5":[1.5,7.5,0.15,null,262144],"mistral-medium-3.5-128b":[1.5,7.5,null,null,256000],"mistral-medium-latest":[1.5,7.5,0.15,null,262144],"mistral-nemo":[0.15,0.15,null,null,131072],"mistral-nemo-base-2407":[0.2,0.2,null,null,128000],"mistral-nemo-instruct-2407":[0.029,0.03,null,null,131072],"mistral-saba-24b":[0.79,0.79,null,null,32768],"mistral-small":[0.1,0.3,0.01,null,32000],"mistral-small-2402":[1.0,3.0,null,null,32000],"mistral-small-24b-instruct-2501":[0.05,0.08,null,null,32768],"mistral-small-2503":[0.1,0.3,null,null,128000],"mistral-small-2603":[0.15,0.6,0.015,null,262144],"mistral-small-3-1-24b-instruct-2503":[0.106,0.318,null,null,131072],"mistral-small-3.1-24b-instruct":[0.351,0.555,null,null,128000],"mistral-small-3.2-24b-instruct-2506":[0.075,0.2,null,null,128000],"mistral-small-4":[0.15,0.6,null,null,262144],"mistral-small-latest":[0.15,0.6,0.015,null,262144],"mistral-tiny":[0.25,0.25,0.025,null,32000],"mistral-vibe-cli-fast":[0.15,0.6,0.015,null,262144],"mistral-vibe-cli-latest":[1.5,7.5,0.15,null,262144],"mistral-vibe-cli-with-tools":[1.5,7.5,0.15,null,262144],"mistral.devstral-2-123b":[0.4,2.0,null,null,256000],"mistral.magistral-small-2509":[0.5,1.5,null,null,128000],"mistral.ministral-3-14b-instruct":[0.2,0.2,null,null,128000],"mistral.ministral-3-3b-instruct":[0.1,0.1,null,null,128000],"mistral.ministral-3-8b-instruct":[0.15,0.15,null,null,128000],"mistral.mixtral-8x7b-instruct":[0.59,0.91,null,null,32000],"mistral.pixtral-large-2502":[2.0,6.0,null,null,128000],"mistral.voxtral-mini-3b-2507":[0.04,0.04,null,null,32768],"mistral.voxtral-small-24b-2507":[0.1,0.3,null,null,32768],"mixtral-8x22b":[1.2,1.2,null,null,65536],"mixtral-8x22b-instruct":[1.2,1.2,null,null,65536],"mixtral-8x22b-instruct-hf":[1.2,1.2,null,null,65536],"mixtral-8x22b-instruct-v0.1":[0.9,0.9,null,null,65536],"mixtral-8x7b":[0.5,0.5,null,null,32768],"mixtral-8x7b-instruct":[0.07,0.28,null,null,4096],"mixtral-8x7b-instruct-hf":[0.5,0.5,null,null,32768],"mixtral-8x7b-instruct-v0.1":[0.15,0.15,null,null,16384],"model-router":[0.14,0.0,null,null,200000],"model_router":[0.14,0,null,null,200000],"moonshot-v1-128k":[2.0,5.0,null,null,131072],"moonshot-v1-128k-vision-preview":[2.0,5.0,null,null,131072],"moonshot-v1-32k":[1.0,3.0,null,null,32768],"moonshot-v1-32k-vision-preview":[1.0,3.0,null,null,32768],"moonshot-v1-8k":[0.2,2.0,null,null,8192],"moonshot-v1-8k-vision-preview":[0.2,2.0,null,null,8192],"moonshot-v1-auto":[2.0,5.0,null,null,131072],"moonshot.kimi-k2-thinking":[0.6,2.5,null,null,256000],"moonshotai.kimi-k2-thinking":[0.73,3.03,null,null,262144],"moonshotai.kimi-k2.5":[0.6,3.0,null,null,256000],"moonshotai.kimi-k3":[3.3,16.5,0.33,4.125,1000000],"moonshotai.kimi-k3.1":[4.4,22.0,0.22,5.5,868928],"morph-dsv41flash":[0.12,0.8,0.008,null,1048576],"morph-dsv41flash-1017":[0.12,0.8,0.008,null,1048576],"morph-dsv41flash-abliterated":[0.3,1.2,0.03,null,1048576],"morph-dsv4flash":[0.141953,0.399625,0.035937,null,1048576],"morph-glm53-744b":[1.19,6.0,0.1955,null,1048576],"morph-glm53flash":[0.2,0.7,0.04,null,1048576],"morph-kimik3":[2.5,14.0,0.29,null,1048576],"morph-kimik3-fast":[6.0,22.5,0.6,null,1048576],"morph-v3-fast":[0.8,1.2,null,null,16000],"morph-v3-large":[0.9,1.9,null,null,16000],"motif-3":[0.5,2.0,0.2,null,262144],"mt0-xxl":[1.908,1.908,null,null,4096],"mt0-xxl-13b":[1.908,1.908,null,null,4096],"multimodalembedding":[0.8,0,null,null,2048],"muse-ember-1.0":[0.1,0.5,0.004,null,1048576],"muse-glimmer-30b":[0.35,1.5,0.04,null,131072],"muse-spark-1-1":[1.25,4.25,1.0,null,1048576],"muse-spark-1-2":[1.25,4.25,1.0,null,1048576],"muse-spark-1-3":[1.25,4.25,1.0,null,1048576],"muse-spark-1.1":[1.25,4.25,0.15,null,1048576],"muse-spark-1.2":[1.25,4.25,0.15,null,1048576],"muse-spark-1.2-contributor":[0.1,0.2,0.002,null,1048576],"muse-spark-1.3":[1.25,4.25,0.15,null,1048576],"muse-spark-1.3-contributor":[0.1,0.2,0.002,null,1048576],"muse-spark-open":[1.25,4.25,0.15,null,1048576],"muse-spark-open-816b-a42b":[1.25,4.25,0.15,null,1048576],"muse-wonder":[5.4,27.0,null,null,1048576],"muse-wonder-1.0":[4.0,20.0,0.2,null,1048576],"mythomax-l2-13b":[0.4,0.4,null,null,4096],"nemotron-120b-a12b":[0.3,0.75,null,null,null],"nemotron-3-120b-a12b":[0.5,1.5,null,null,256000],"nemotron-3-nano-30b-a3b":[0.05,0.2,null,null,262144],"nemotron-3-nano-omni":[0.06,0.24,null,null,262144],"nemotron-3-super-120b-a12b":[0.3,0.9,0.06,null,262144],"nemotron-3-ultra-550b":[0.675,3.375,0.135,null,1000000],"nemotron-3-ultra-550b-a55b":[1.0,3.0,0.12,null,1048576],"nemotron-3-ultra-nvfp4":[0.6,2.4,0.12,null,262144],"nemotron-3.5-lightning-30b-a3b":[0.0115,0.17,0.00115,null,null],"nemotron-3.5-super-120b-a12b":[0.2,0.5,0.04,null,null],"nemotron-3.5-super-vl":[0.081,0.4725,0.0135,null,1048576],"nemotron-3_5-lightning":[0.06,0.24,0.03,null,1048576],"nemotron-content-safety-3.5":[0.2,0.2,null,null,131072],"nemotron-lightning-3p5-30b-a3b":[0.05,0.2,0.01,null,262144],"nemotron-nano-v2-12b-vl":[0.1,0.1,null,null,131072],"nomic-embed-text":[0.02,0,null,null,8192],"nomic-embed-text-v1":[0.008,0.0,null,null,8192],"nomic-embed-text-v1.5":[0.008,0.0,null,null,8192],"nous-capybara-7b-v1p9":[0.2,0.2,null,null,32768],"nous-hermes-2-mixtral-8x7b-dpo":[0.5,0.5,null,null,32768],"nous-hermes-2-yi-34b":[0.9,0.9,null,null,4096],"nous-hermes-llama2-13b":[0.2,0.2,null,null,4096],"nous-hermes-llama2-70b":[0.9,0.9,null,null,4096],"nous-hermes-llama2-7b":[0.2,0.2,null,null,4096],"nova-2-lite":[0.3,2.5,0.075,null,1000000],"nova-2-multimodal-embeddings":[0.135,0.0,null,null,8172],"nova-2-pro-preview-20251202":[1.25,10.0,0.3125,null,1000000],"nova-lite":[0.06,0.24,0.015,null,300000],"nova-lite-1-0":[0.069,0.28,0.0386,null,317379],"nova-lite-2":[0.38,3.16,0.2128,null,1000000],"nova-lite-v1":[0.06,0.24,null,null,300000],"nova-micro":[0.035,0.14,0.00875,null,128000],"nova-micro-1-0":[0.04,0.16,0.0224,null,133003],"nova-micro-v1":[0.035,0.14,null,null,128000],"nova-premier":[2.5,12.5,0.625,null,1000000],"nova-premier-v1":[2.5,12.5,null,null,1000000],"nova-pro":[0.8,3.2,0.2,null,300000],"nova-pro-1-0":[2.4,9.6,null,null,317319],"nova-pro-v1":[0.8,3.2,null,null,300000],"nvidia-nemotron-3-nano-30b-a3b":[0.06,0.24,0.012,null,262144],"nvidia-nemotron-3-super-120b-a12b":[0.075,0.4,null,null,262144],"nvidia-nemotron-3-ultra-550b-a55b":[0.5,2.15,0.1,null,262000],"nvidia-nemotron-3.5-lightning":[0.06,0.16,0.04,null,262144],"nvidia-nemotron-3.5-lightning-30b-a3b":[0.07,0.2,0.04,null,262000],"nvidia-nemotron-3.5-lightning-30b-a3b-bf16":[0.15,1.0,0.05,null,65536],"nvidia-nemotron-3.5-super-vl":[0.15,0.5,0.02,null,262144],"nvidia-nemotron-3.5-super-vl-120b-a12b":[0.2,0.5,0.04,null,1048576],"nvidia-nemotron-3.5-super-vl-120b-a12b-bf16":[0.1,0.5,0.05,null,32768],"nvidia-nemotron-nano-12b-v2":[0.2,0.2,null,null,128000],"nvidia-nemotron-nano-9b-v2":[0.04,0.16,null,null,131072],"nvidia.nemotron-nano-12b-v2":[0.2,0.6,null,null,128000],"nvidia.nemotron-nano-3-30b":[0.06,0.24,null,null,256000],"nvidia.nemotron-nano-9b-v2":[0.06,0.23,null,null,128000],"nvidia.nemotron-super-3-120b":[0.15,0.65,null,null,256000],"o1":[15.0,60.0,7.5,null,200000],"o1-2024-12-17":[15.0,60.0,7.5,null,200000],"o1-mini":[1.1,4.4,0.55,null,128000],"o1-mini-2024-09-12":[1.21,4.84,0.605,null,128000],"o1-pro":[150.0,600.0,null,null,200000],"o1-pro-2025-03-19":[150.0,600.0,null,null,200000],"o3":[2.0,8.0,0.5,null,200000],"o3-2025-04-16":[2.0,8.0,0.5,null,200000],"o3-deep-research":[10.0,40.0,2.5,null,200000],"o3-deep-research-2025-06-26":[10.0,40.0,2.5,null,200000],"o3-mini":[1.1,4.4,0.55,null,200000],"o3-mini-2025-01-31":[1.1,4.4,0.55,null,200000],"o3-pro":[20.0,80.0,null,null,200000],"o3-pro-2025-06-10":[20.0,80.0,null,null,200000],"o4-mini":[1.1,4.4,0.275,null,200000],"o4-mini-2025-04-16":[1.1,4.4,0.275,null,200000],"o4-mini-deep-research":[2.0,8.0,0.5,null,200000],"o4-mini-deep-research-2025-06-26":[2.0,8.0,0.5,null,200000],"olmo-3-32b-think":[0.0,0.0,null,null,32768],"olmo-3-7b-instruct":[0.0,0.0,null,null,32768],"olmo-3-7b-think":[0.0,0.0,null,null,32768],"olmocr-7b-0725-fp8":[0.27,1.5,null,null,16384],"open-mistral-nemo":[0.3,0.3,0.03,null,128000],"openai-gpt-4.1":[2.0,8.0,0.5,null,300000],"openai-gpt-5":[1.25,10.0,0.125,null,300000],"openai-gpt-5-mini":[0.3,1.2,null,null,1000000],"openai-gpt-5-nano":[0.15,0.6,null,null,5000000],"openai-o3":[2.0,8.0,null,null,200000],"openai-o3-mini":[1.1,4.4,null,null,200000],"openai.gpt-5":[1.25,10.0,null,null,272000],"openai.gpt-5-mini":[0.25,2.0,null,null,272000],"openai.gpt-5-nano":[0.05,0.4,null,null,272000],"openai.gpt-5.4":[2.75,16.5,0.275,null,922000],"openai.gpt-5.5":[5.5,33.0,0.55,null,922000],"openai.gpt-5.6-cyber":[13.75,82.5,1.375,17.1875,922000],"openai.gpt-5.6-luna":[0.22,1.32,0.022,0.275,922000],"openai.gpt-5.6-sol":[4.4,22.0,0.44,5.5,922000],"openai.gpt-5.6-terra":[2.2,13.2,0.22,2.75,922000],"openai.gpt-6-astra":[11.0,55.0,1.1,13.75,922000],"openai.gpt-6-luna":[0.1,0.5,0.01,0.125,922000],"openai.gpt-6-sol":[2.0,10.0,0.2,2.5,922000],"openai.gpt-6.1-sol":[2.2,11.0,0.11,2.75,1000000],"openai.gpt-6.1-terra":[1.1,5.5,0.055,1.375,922000],"openai.gpt-daybreak-blue-5.6-sol":[4.4,22.0,0.44,5.5,922000],"openai.gpt-oss-120b":[0.15,0.6,null,null,128000],"openai.gpt-oss-120b-1":[0.15,0.6,null,null,128000],"openai.gpt-oss-20b":[0.07,0.3,null,null,128000],"openai.gpt-oss-20b-1":[0.07,0.3,null,null,128000],"openai.gpt-oss-safeguard-120b":[0.15,0.6,null,null,128000],"openai.gpt-oss-safeguard-20b":[0.07,0.2,null,null,128000],"openchat-3p5-0106-7b":[0.2,0.2,null,null,8192],"openhermes-2-mistral-7b":[0.2,0.2,null,null,32768],"openhermes-2p5-mistral-7b":[0.2,0.2,null,null,32768],"openorca-7b":[0.2,0.2,null,null,32768],"openthinker-7b":[0.08,0.15,null,null,32768],"orca-mini":[0.0,0.0,null,null,4096],"paddleocr-vl":[0.02,0.02,null,null,16384],"perplexity-advanced-deep-research":[12.0,60.0,null,null,null],"perplexity-deep-research":[4.8,19.0,null,null,919327],"perplexity-pro-search":[7.8,39.0,null,null,null],"perplexity-sonar":[2.4,2.4,null,null,919327],"perplexity-sonar-pro":[7.2,36.0,null,null,919328],"perplexity-sonar-reasoning-pro":[4.8,19.0,null,null,919330],"phi-2-3b":[0.1,0.1,null,null,2048],"phi-3-mini-128k-instruct":[0.1,0.1,null,null,131072],"phi-3-vision-128k-instruct":[0.2,0.2,null,null,32064],"phi-4":[0.125,0.5,null,null,16384],"phi-4-mini-instruct":[0.075,0.3,null,null,131072],"phi-4-mini-reasoning":[0.075,0.3,null,null,131072],"phi-4-multimodal-instruct":[0.08,0.32,null,null,131072],"phi-4-reasoning":[0.125,0.5,null,null,32768],"phind-code-llama-34b-python-v1":[0.9,0.9,null,null,16384],"phind-code-llama-34b-v1":[0.9,0.9,null,null,16384],"phind-code-llama-34b-v2":[0.9,0.9,null,null,16384],"pixtral-12b":[0.15,0.15,null,null,128000],"pixtral-12b-2409":[0.35,0.35,null,null,128000],"pixtral-large":[2.0,6.0,null,null,128000],"pixtral-large-latest":[2.0,6.0,0.2,null,128000],"pplx-70b-chat":[0.7,2.8,null,null,4096],"pplx-70b-online":[0.0,2.8,null,null,4096],"pplx-7b-chat":[0.07,0.28,null,null,8192],"pplx-7b-online":[0.0,0.28,null,null,4096],"pplx-embed-context-v1-0.6b":[0.008,0.0,null,null,32768],"pplx-embed-context-v1-4b":[0.05,0.0,null,null,32768],"pplx-embed-v1-0.6b":[0.004,0.0,null,null,32768],"pplx-embed-v1-4b":[0.03,0.0,null,null,32768],"pythia-12b":[0.2,0.2,null,null,2048],"query":[0.4,3.5,null,null,null],"qwen-2-1.5b-instruct":[0.1,0.1,null,null,32768],"qwen-2.5-72b-instruct":[0.38,0.4,null,null,32000],"qwen-3-14b":[0.08,0.24,null,null,40960],"qwen-3-235b":[0.2,0.6,null,null,40960],"qwen-3-30b":[0.1,0.3,null,null,40960],"qwen-3-32b":[0.4,0.8,null,null,128000],"qwen-3-coder":[0.3546,1.612,0.135,null,131072],"qwen-3.6-plus":[0.4388,2.633,null,null,1000000],"qwen-3.7-max":[2.071,6.207,0.16875,null,1000000],"qwen-3.7-plus":[0.4325,1.731,0.0864,null,1000000],"qwen-3.8-27b":[0.99,1.49,null,null,65536],"qwen-3.8-max":[2.228,6.684,0.2781,null,1000000],"qwen-coder":[0.3,1.5,null,null,1000000],"qwen-max":[1.6,6.4,null,null,30720],"qwen-max-latest":[2.0,6.0,0.25,null,262144],"qwen-mt-plus":[0.25,0.75,null,null,16384],"qwen-plus":[0.4,1.2,null,null,129024],"qwen-plus-2025-01-25":[0.4,1.2,null,null,129024],"qwen-plus-2025-04-28":[0.4,1.2,null,null,129024],"qwen-plus-2025-07-14":[0.4,1.2,null,null,129024],"qwen-qwq-32b-preview":[0.9,0.9,null,null,32768],"qwen-sea-lion-v4-32b-it":[0.0,0.0,null,null,32768],"qwen-turbo":[0.05,0.2,null,null,129024],"qwen-turbo-2024-11-01":[0.05,0.2,null,null,1000000],"qwen-turbo-2025-04-28":[0.05,0.2,null,null,1000000],"qwen-turbo-latest":[0.05,0.2,null,null,1000000],"qwen-v2p5-14b-instruct":[0.2,0.2,null,null,32768],"qwen-v2p5-7b":[0.2,0.2,null,null,131072],"qwen.qwen3-235b-a22b-2507":[0.22,0.88,null,null,262144],"qwen.qwen3-32b":[0.15,0.6,null,null,32768],"qwen.qwen3-coder-30b-a3b":[0.15,0.6,null,null,262144],"qwen.qwen3-coder-30b-a3b-instruct":[0.15,0.6,null,null,256000],"qwen.qwen3-coder-480b-a35b":[0.45,1.8,null,null,262144],"qwen.qwen3-coder-480b-a35b-instruct":[0.45,1.8,null,null,262144],"qwen.qwen3-coder-next":[0.5,1.2,null,null,256000],"qwen.qwen3-next-80b-a3b":[0.15,1.2,null,null,256000],"qwen.qwen3-next-80b-a3b-instruct":[0.14,1.2,null,null,256000],"qwen.qwen3-vl-235b-a22b":[0.53,2.66,null,null,256000],"qwen.qwen3-vl-235b-a22b-instruct":[0.53,2.66,null,null,256000],"qwen1p5-72b-chat":[0.9,0.9,null,null,32768],"qwen2-1.5b-instruct":[0.02,0.02,null,null,32768],"qwen2-72b-instruct":[0.9,0.9,null,null,32768],"qwen2-7b-instruct":[0.2,0.2,null,null,32768],"qwen2-vl-2b-instruct":[0.1,0.1,null,null,32768],"qwen2-vl-72b-instruct":[0.13,0.4,null,null,131072],"qwen2-vl-7b-instruct":[0.02,0.06,null,null,131072],"qwen2.5-14b-instruct":[0.8,0.8,null,null,32768],"qwen2.5-32b-instruct":[0.06,0.2,null,null,128000],"qwen2.5-72b-instruct":[0.36,0.4,null,null,32768],"qwen2.5-72b-instruct-turbo":[1.2,1.2,null,null,131072],"qwen2.5-7b-instruct":[0.04,0.1,null,null,32768],"qwen2.5-7b-instruct-turbo":[0.3,0.3,null,null,32768],"qwen2.5-coder-32b-instruct":[0.66,1.0,null,null,32768],"qwen2.5-coder-3b-instruct":[0.01,0.03,null,null,null],"qwen2.5-coder-7b":[0.01,0.03,null,null,32768],"qwen2.5-coder-7b-instruct":[0.01,0.03,null,null,null],"qwen2.5-vl-32b-instruct":[0.2,0.6,null,null,128000],"qwen2.5-vl-72b-instruct":[0.25,0.75,null,null,32000],"qwen25-coder-32b-instruct":[0.05,0.1,null,null,131072],"qwen2p5-0p5b-instruct":[0.1,0.1,null,null,32768],"qwen2p5-14b":[0.2,0.2,null,null,131072],"qwen2p5-1p5b-instruct":[0.1,0.1,null,null,32768],"qwen2p5-32b":[0.9,0.9,null,null,131072],"qwen2p5-32b-instruct":[0.9,0.9,null,null,32768],"qwen2p5-72b":[0.9,0.9,null,null,131072],"qwen2p5-72b-instruct":[0.9,0.9,null,null,32768],"qwen2p5-7b-instruct":[0.2,0.2,null,null,32768],"qwen2p5-coder-0p5b":[0.1,0.1,null,null,32768],"qwen2p5-coder-0p5b-instruct":[0.1,0.1,null,null,32768],"qwen2p5-coder-14b":[0.2,0.2,null,null,32768],"qwen2p5-coder-14b-instruct":[0.2,0.2,null,null,32768],"qwen2p5-coder-1p5b":[0.1,0.1,null,null,32768],"qwen2p5-coder-1p5b-instruct":[0.1,0.1,null,null,32768],"qwen2p5-coder-32b":[0.9,0.9,null,null,32768],"qwen2p5-coder-32b-instruct":[0.9,0.9,null,null,4096],"qwen2p5-coder-32b-instruct-128k":[0.9,0.9,null,null,131072],"qwen2p5-coder-32b-instruct-32k-rope":[0.9,0.9,null,null,32768],"qwen2p5-coder-32b-instruct-64k":[0.9,0.9,null,null,65536],"qwen2p5-coder-3b":[0.1,0.1,null,null,32768],"qwen2p5-coder-3b-instruct":[0.1,0.1,null,null,32768],"qwen2p5-coder-7b":[0.2,0.2,null,null,32768],"qwen2p5-coder-7b-instruct":[0.2,0.2,null,null,32768],"qwen2p5-math-72b-instruct":[0.9,0.9,null,null,4096],"qwen2p5-vl-32b-instruct":[0.9,0.9,null,null,128000],"qwen2p5-vl-3b-instruct":[0.2,0.2,null,null,128000],"qwen2p5-vl-72b-instruct":[0.9,0.9,null,null,128000],"qwen2p5-vl-7b-instruct":[0.2,0.2,null,null,128000],"qwen3-0p6b":[0.1,0.1,null,null,40960],"qwen3-14b":[0.12,0.24,null,null,40960],"qwen3-14b-instruct":[0.05,0.22,null,null,32800],"qwen3-1p7b":[0.1,0.1,null,null,131072],"qwen3-1p7b-fp8-draft":[0.1,0.1,null,null,262144],"qwen3-1p7b-fp8-draft-131072":[0.1,0.1,null,null,131072],"qwen3-1p7b-fp8-draft-40960":[0.1,0.1,null,null,40960],"qwen3-235b-a22b":[0.18,0.54,null,null,40960],"qwen3-235b-a22b-fp8":[0.2,0.8,null,null,40960],"qwen3-235b-a22b-instruct-2507":[0.09,0.55,null,null,262144],"qwen3-235b-a22b-instruct-2507-fp8":[0.0875,0.35,0.0175,null,262144],"qwen3-235b-a22b-instruct-2507-maas":[0.22,0.88,null,null,262144],"qwen3-235b-a22b-thinking-2507":[0.3,2.9,null,null,262144],"qwen3-235b-a22b-thinking-2507-fp8":[0.35,4.2,null,null,262141],"qwen3-30b-a3b":[0.12,0.5,null,null,40960],"qwen3-30b-a3b-fp8":[0.0509,0.335,null,null,32768],"qwen3-30b-a3b-instruct-2507":[0.1,0.3,0.1,null,262144],"qwen3-30b-a3b-thinking-2507":[0.9,0.9,null,null,262144],"qwen3-32b":[0.08,0.28,null,null,40960],"qwen3-32b-fp8":[0.05,0.1,null,null,131072],"qwen3-4b":[0.08,0.24,null,null,32768],"qwen3-4b-fp8":[0.03,0.03,null,null,128000],"qwen3-4b-instruct-2507":[0.2,0.2,null,null,262144],"qwen3-4b-instruct-2507-gguf":[0,0,null,null,262144],"qwen3-5-122b-a10b":[0.115,0.917,null,null,258048],"qwen3-5-27b":[0.086,0.688,null,null,258048],"qwen3-5-35b-a3b":[0.057,0.459,null,null,258048],"qwen3-5-397b-a17b":[0.172,1.032,null,null,258048],"qwen3-5-4b":[0.04,0.07,0.02,null,262144],"qwen3-5-9b":[0.09,0.13,0.045,null,262144],"qwen3-5-flash":[0.09,0.368,null,null,995904],"qwen3-5-omni-flash":[0.8,4.4,null,null,196608],"qwen3-5-omni-plus":[2.8,16.6,null,null,196608],"qwen3-5-plus":[0.36,2.21,null,null,983616],"qwen3-6-27b":[0.412564,2.475384,null,null,229376],"qwen3-6-35b-a3b":[0.07,0.42,0.035,null,131072],"qwen3-6-flash":[0.25,1.5,null,null,983616],"qwen3-6-plus":[0.5,3.0,null,null,983616],"qwen3-7-flash":[0.03,0.13,0.006,null,983616],"qwen3-7-max":[2.5,7.5,null,null,983616],"qwen3-7-plus":[0.4,1.6,null,null,983616],"qwen3-8-27b":[0.17,0.5,0.08,null,262144],"qwen3-8-flash":[0.16,0.47,null,null,983616],"qwen3-8-max":[2.0,6.0,null,null,983616],"qwen3-8-max-0902":[2.0,6.0,null,null,983616],"qwen3-8-omni-flash":[0.3,0.94,null,null,1006592],"qwen3-8b":[0.2,0.2,null,null,40960],"qwen3-8b-fp8":[0.035,0.138,null,null,128000],"qwen3-coder":[0.0,0.0,null,null,262144],"qwen3-coder-30b-a3b":[0.3,0.6,null,null,131072],"qwen3-coder-30b-a3b-instruct":[0.15,0.6,null,null,262144],"qwen3-coder-30b-a3b-instruct-gguf":[0,0,null,null,262144],"qwen3-coder-480b-a35b-instruct":[0.4,1.6,null,null,262144],"qwen3-coder-480b-a35b-instruct-fp8":[0.45,1.8,0,null,262144],"qwen3-coder-480b-a35b-instruct-maas":[0.22,1.8,0.022,null,262144],"qwen3-coder-480b-a35b-instruct-turbo":[0.3,1.0,0.1,null,262144],"qwen3-coder-480b-instruct-bf16":[0.9,0.9,null,null,262144],"qwen3-coder-next":[0.2,1.5,null,null,262144],"qwen3-coder-next-fp8":[0.5,1.2,null,null,262144],"qwen3-embedding-0.6b":[0.07,0,null,null,32768],"qwen3-embedding-0p6b":[0.0,0.0,null,null,32768],"qwen3-embedding-4b":[0.0,0.0,null,null,40960],"qwen3-embedding-8b":[0.01,0.0,null,null,40960],"qwen3-max":[2.11,8.45,null,null,262144],"qwen3-max-thinking":[1.2,6.0,0.24,null,256000],"qwen3-next-80b-a3b-instruct":[0.15,1.2,null,null,262144],"qwen3-next-80b-a3b-instruct-maas":[0.15,1.2,null,null,262144],"qwen3-next-80b-a3b-thinking":[0.15,1.2,null,null,262144],"qwen3-next-80b-a3b-thinking-maas":[0.15,1.2,null,null,262144],"qwen3-omni-30b-a3b-instruct":[0.25,0.97,null,null,65536],"qwen3-omni-30b-a3b-thinking":[0.25,0.97,null,null,65536],"qwen3-vl":[0.26,1.15,null,null,262144],"qwen3-vl-235b":[0.26,1.15,null,null,262144],"qwen3-vl-235b-a22b-instruct":[0.4,1.6,null,null,131072],"qwen3-vl-235b-a22b-instruct-fp8":[0.3,1.4,null,null,262144],"qwen3-vl-235b-a22b-thinking":[0.4,4.0,null,null,131072],"qwen3-vl-30b-a3b-instruct":[0.15,0.6,null,null,262144],"qwen3-vl-30b-a3b-thinking":[0.15,0.6,null,null,262144],"qwen3-vl-32b-instruct":[0.16,0.64,null,null,131072],"qwen3-vl-32b-thinking":[0.16,2.87,null,null,131072],"qwen3-vl-8b":[0.15,0.55,null,null,32768],"qwen3-vl-8b-instruct":[0.2,0.2,null,null,262144],"qwen3.5-122b-a10b":[0.25,1.75,null,null,262144],"qwen3.5-122b-a10b-thinking":[0.25,1.75,null,null,262144],"qwen3.5-27b":[0.3,2.4,null,null,262144],"qwen3.5-35b-a3b":[0.25,2.0,null,null,262144],"qwen3.5-397b-a17b":[0.6,3.6,0.12,null,262144],"qwen3.5-397b-a17b-fp8":[0.6,3.6,0,null,262144],"qwen3.5-9b":[0.17,0.25,null,null,262144],"qwen3.6-27b":[0.15,0.5,null,null,262144],"qwen3.6-27b-fp8":[0.32,3.2,0,null,262144],"qwen3.6-27b-thinking":[0.15,0.5,null,null,262144],"qwen3.6-35b-a3b":[0.05,0.4,0.02,null,262144],"qwen3.6-35b-a3b-thinking":[0.15,0.5,null,null,262144],"qwen3.6-max-preview":[1.268,7.608,0.1268,1.585,262144],"qwen3.6-plus":[0.5,3.0,null,null,1000000],"qwen3.6-plus-2026-04-02":[0.5,3.0,null,null,262144],"qwen3.7-flash":[0.0435,0.1886,0.0081,null,1000000],"qwen3.7-max":[2.5,7.5,0.5,3.125,991808],"qwen3.7-plus":[0.32,1.28,null,null,1000000],"qwen3.8-2.4t-a95b":[2.0,6.0,0.25,null,1010000],"qwen3.8-27b":[0.8,4.0,null,null,131072],"qwen3.8-flash":[0.15,0.47,0.016,0.2,991808],"qwen3.8-max":[2.0,6.0,0.25,null,991808],"qwen3.8-max-0902":[1.65,4.951,0.206,null,262144],"qwen3.8-omni-flash":[0.15,0.47,0.016,null,991808],"qwen3p7-plus":[0.4,1.6,0.08,null,262144],"qwen3p8-max":[2.0,6.0,0.25,null,262144],"qwq-32b":[0.66,1.0,null,null,24000],"qwq-plus":[0.8,2.4,null,null,98304],"r1v4-lite":[0.2,0.6,null,null,262144],"ripple-65b-preview":[0.2,0.6,0.05,null,1000000],"rnj-1-instruct":[0.15,0.15,null,null,null],"rolm-ocr":[0.2,0.2,null,null,128000],"salamandra-7b-instruct-tools-16k":[0.0,0.0,null,null,16384],"sarvam-m":[0,0,0,0,8192],"seed-1.8":[0.25,2.0,0.05,null,256000],"seed-2-0-code":[0.4,2.4,null,null,229096],"seed-2-0-lite":[0.31,2.5,null,null,229096],"seed-2-0-mini":[0.12,0.5,null,null,229156],"seed-2-0-pro":[0.63,3.79,null,null,229096],"seed-2-1-turbo":[0.63,3.13,null,null,229093],"seed-2.0-code":[0.5,3.0,0.1,null,256000],"seed-2.0-mini":[0.1,0.4,0.02,null,256000],"seed-2.0-pro":[0.5,3.0,0.1,null,256000],"skylark-embedding-vision":[0.25,0,null,null,8000],"snorkel-mistral-7b-pairrm-dpo":[0.2,0.2,null,null,32768],"snowflake-arctic-embed-l-v2.0":[0.07,0.0,null,null,8192],"snowflake-arctic-embed-m-v2.0":[0.07,0.0,null,null,8192],"snowflake-arctic-instruct":[20.0,20.0,null,null,null],"snowflake-llama-3.3-70b":[0.72,0.72,null,null,128000],"sonar":[1.0,1.0,null,null,128000],"sonar-deep-research":[2.0,8.0,null,null,128000],"sonar-medium-chat":[0.6,1.8,null,null,16384],"sonar-medium-online":[0,1.8,null,null,12000],"sonar-pro":[3.0,15.0,null,null,200000],"sonar-reasoning":[1.0,5.0,null,null,128000],"sonar-reasoning-pro":[2.0,8.0,null,null,128000],"sonar-small-chat":[0.07,0.28,null,null,16384],"sonar-small-online":[0,0.28,null,null,12000],"spot-deepseek-v4-flash":[0.105,0.21,0.021,null,1048576],"stablecode-3b":[0.1,0.1,null,null,4096],"starcoder-16b":[0.2,0.2,null,null,8192],"starcoder-7b":[0.2,0.2,null,null,8192],"starcoder2-15b":[0.2,0.2,null,null,16384],"starcoder2-3b":[0.1,0.1,null,null,16384],"starcoder2-7b":[0.2,0.2,null,null,16384],"step-3-5-flash":[0.1,0.3,0.02,null,262144],"step-3-5-flash-2603":[0.1,0.3,0.02,null,262144],"step-3-7-flash":[0.2,1.15,0.04,null,262144],"step-3.7-flash":[0.2,1.15,0.04,null,262144],"step-5-preview":[1.0,2.7,0.1,null,1000000],"stepaudio-2-5-chat":[1.5,3.5,0.3,null,262144],"stepaudio-3-chat":[1.5,3.5,0.3,null,262144],"summarize":[0.05,0.0,null,null,null],"swe-1.6":[0.5,2.5,0.2,null,null],"swe-1.7":[0.5,2.5,0.2,null,null],"swe-1.7-lightning":[2.5,12.5,1.0,null,null],"system.ai.gemini-4-carbon":[2.0,10.0,2.0,2.0,1048576],"ternary-bonsai-27b":[0.0,0.0,null,null,262144],"tev1-4b-experimental":[0.042,0.0,null,null,32768],"text-embedding-004":[0.1,0,null,null,2048],"text-embedding-005":[0.1,0,null,null,2048],"text-embedding-3-large":[0.13,0.0,null,null,8191],"text-embedding-3-small":[0.02,0.0,null,null,8191],"text-embedding-ada-002":[0.1,0.0,null,null,8191],"text-embedding-ada-002-v2":[0.1,0.0,null,null,8191],"text-embedding-large-exp-03-07":[0.1,0,null,null,8192],"text-embedding-preview-0409":[0.00625,0,null,null,3072],"text-embedding-v4":[0.07,0,null,null,8192],"text-multilingual-embedding-002":[0.1,0,null,null,2048],"text-unicorn":[10.0,28.0,null,null,8192],"titan-embed-g1-text-02":[0.1,0.0,null,null,8192],"titan-embed-image-v1":[0.8,0.0,null,null,128],"titan-embed-text":[0.02,0.0,null,null,8192],"titan-embed-text-v1":[0.1,0.0,null,null,8192],"titan-embed-text-v2":[0.02,0.0,null,null,0],"titan-text-express-v1":[0.2,0.6,null,null,42000],"titan-text-lite-v1":[0.15,0.2,null,null,42000],"titan-text-premier":[0.5,1.5,null,null,42000],"together-ai-21.1b-41b":[0.8,0.8,null,null,null],"together-ai-4.1b-8b":[0.2,0.2,null,null,null],"together-ai-41.1b-80b":[0.9,0.9,null,null,null],"together-ai-8.1b-21b":[0.3,0.3,null,null,1000],"together-ai-81.1b-110b":[1.8,1.8,null,null,null],"together-ai-embedding-151m-to-350m":[0.016,0.0,null,null,null],"together-ai-embedding-up-to-150m":[0.008,0.0,null,null,null],"together-ai-up-to-4b":[0.1,0.1,null,null,null],"tongyi-embedding-vision-flash":[0.09,0,null,null,1024],"tongyi-embedding-vision-plus":[0.09,0,null,null,1024],"toppy-m-7b":[0.2,0.2,null,null,32768],"trinity-mini":[0.045,0.15,null,null,128000],"uae-large-v1":[0.016,0.0,null,null,512],"ultramarine-1-flash-us":[0.15,0.5,0.03,null,1048576],"us-gov.nvidia.nemotron-nano-12b-v2":[0.24,0.72,null,null,128000],"us-gov.nvidia.nemotron-nano-3-30b":[0.072,0.288,null,null,262144],"us-gov.nvidia.nemotron-nano-9b-v2":[0.072,0.276,null,null,128000],"us-gov.nvidia.nemotron-super-3-120b":[0.18,0.78,null,null,256000],"us-gov.openai.gpt-oss-120b-1":[0.18,0.72,null,null,128000],"us-gov.openai.gpt-oss-20b-1":[0.084,0.36,null,null,128000],"us-gov.xai.grok-4.6":[2.64,7.92,0.66,null,500000],"us-gov.xai.grok-4.7":[2.64,7.92,0.66,null,524288],"us.cohere.embed":[0.12,0.0,null,null,128000],"us.deepseek.r1":[1.35,5.4,null,null,128000],"us.deepseek.v3.2":[0.62,1.85,null,null,163840],"us.moonshotai.kimi-k3":[3.3,16.5,0.33,4.125,1000000],"us.moonshotai.kimi-k3.1":[4.4,22.0,0.22,5.5,868928],"us.openai.gpt-5.4":[2.75,16.5,0.275,null,922000],"us.openai.gpt-5.5":[5.5,33.0,0.55,null,922000],"us.openai.gpt-5.6-luna":[0.22,1.32,0.022,0.275,922000],"us.openai.gpt-5.6-sol":[4.4,22.0,0.44,5.5,922000],"us.openai.gpt-5.6-terra":[2.2,13.2,0.22,2.75,922000],"us.openai.gpt-6-astra":[11.0,55.0,1.1,13.75,922000],"us.openai.gpt-6-luna":[0.11,0.55,0.011,0.1375,922000],"us.openai.gpt-6-sol":[2.2,11.0,0.22,2.75,922000],"us.openai.gpt-6.1-sol":[2.2,11.0,0.11,2.75,1000000],"us.openai.gpt-6.1-terra":[1.1,5.5,0.055,1.375,922000],"us.writer.palmyra-x4":[2.5,10.0,null,null,128000],"us.writer.palmyra-x5":[0.6,6.0,null,null,128000],"us.xai.grok-4.6":[2.2,6.6,0.55,null,500000],"us.xai.grok-4.7":[2.2,6.6,0.55,null,524288],"us.zai.glm-5.3":[1.848,5.808,0.3432,2.31,1000000],"v0-1.0-md":[3.0,15.0,null,null,128000],"v0-1.5-lg":[15.0,75.0,null,null,512000],"v0-1.5-md":[3.0,15.0,null,null,128000],"vicuna":[0.0,0.0,null,null,2048],"voxtral-small-2507":[0.1,0.4,0.01,null,32768],"voxtral-small-latest":[0.1,0.4,0.01,null,32768],"voyage-2":[0.1,0.0,null,null,4000],"voyage-3":[0.06,0.0,null,null,32000],"voyage-3-large":[0.18,0.0,null,null,32000],"voyage-3-lite":[0.02,0.0,null,null,32000],"voyage-3.5":[0.06,0.0,null,null,32000],"voyage-3.5-lite":[0.02,0.0,null,null,32000],"voyage-4":[0.06,0.0,null,null,32000],"voyage-4-large":[0.12,0.0,null,null,32000],"voyage-4-lite":[0.02,0.0,null,null,32000],"voyage-code-2":[0.12,0.0,null,null,16000],"voyage-code-3":[0.18,0.0,null,null,32000],"voyage-code-4":[0.12,0.0,null,null,32000],"voyage-context-3":[0.18,0.0,null,null,120000],"voyage-context-4":[0.12,0.0,null,null,120000],"voyage-finance-2":[0.12,0.0,null,null,32000],"voyage-large-2":[0.12,0.0,null,null,16000],"voyage-large-2-instruct":[0.12,0.0,null,null,16000],"voyage-law-2":[0.12,0.0,null,null,16000],"voyage-lite-01":[0.1,0.0,null,null,4000],"voyage-lite-02-instruct":[0.1,0.0,null,null,4000],"voyage-multilingual-2":[0.12,0.0,null,null,32000],"voyage-multimodal-3":[0.12,0.0,null,null,32000],"voyage-multimodal-3.5":[0.12,0.0,null,null,32000],"wizardlm-2-8x22b":[0.48,0.48,null,null,65536],"writer.palmyra-vision-7b":[0.15,0.6,null,null,4096],"writer.palmyra-x4":[2.5,10.0,null,null,128000],"writer.palmyra-x5":[0.6,6.0,null,null,128000],"xai.grok-3":[3.0,15.0,null,null,131072],"xai.grok-3-fast":[5.0,25.0,null,null,131072],"xai.grok-3-mini":[0.3,0.5,null,null,131072],"xai.grok-3-mini-fast":[0.6,4.0,null,null,131072],"xai.grok-4":[3.0,15.0,null,null,128000],"xai.grok-4-fast":[5.0,25.0,null,null,131072],"xai.grok-4.1-fast":[5.0,25.0,null,null,131072],"xai.grok-4.20":[3.0,15.0,null,null,131072],"xai.grok-4.20-0309-non-reasoning":[1.25,2.5,0.2,null,1000000],"xai.grok-4.20-0309-reasoning":[1.25,2.5,0.2,null,1000000],"xai.grok-4.20-multi-agent":[3.0,15.0,null,null,131072],"xai.grok-4.20-non-reasoning":[1.25,2.5,0.2,null,1000000],"xai.grok-4.20-reasoning":[1.25,2.5,0.2,null,1000000],"xai.grok-4.3":[1.25,2.5,0.2,null,1048576],"xai.grok-4.6":[2.2,6.6,0.55,null,500000],"xai.grok-4.7":[2.64,7.92,0.66,null,524288],"xai.grok-code-fast-1":[5.0,25.0,null,null,131072],"yi-34b":[0.9,0.9,null,null,4096],"yi-34b-200k-capybara":[0.9,0.9,null,null,200000],"yi-34b-chat":[0.9,0.9,null,null,4096],"yi-6b":[0.2,0.2,null,null,4096],"yi-large":[3.0,3.0,null,null,32768],"zai-glm-5":[1.4,4.4,0.14,null,1048576],"zai-glm-5-2":[1.4,4.4,0.14,null,1048576],"zai-glm-5-3":[1.4,4.4,0.14,null,1048576],"zai-glm-5-3-flash":[0.15,0.5,0.03,null,1048576],"zai-glm-latest":[1.4,4.4,0.14,null,1048576],"zai.glm-4.7":[0.6,2.2,null,null,203000],"zai.glm-4.7-flash":[0.07,0.4,null,null,203000],"zai.glm-5":[1.0,3.2,null,null,200000],"zephyr-7b-beta":[0.15,0.15,null,null,16384]} diff --git a/static/public/dashboards/llm-observability.yaml b/static/public/dashboards/llm-observability.yaml new file mode 100644 index 000000000..e04449552 --- /dev/null +++ b/static/public/dashboards/llm-observability.yaml @@ -0,0 +1,108 @@ +title: LLM & Agents +tags: [llm, ai, agents, genai, opentelemetry] +read_only: true +icon: sparkles +preview: /public/assets/svgs/screens/http-stats.svg +description: Model calls, tokens, cost, latency and tool use from OpenTelemetry GenAI spans, whichever SDK or framework sent them +refresh_interval: 60s +# Token and latency figures read inference spans only: agent spans may carry their run's aggregate +# usage, which would count every call twice. Cost is only ever stamped on inference spans. +widgets: + - type: group + title: Overview + layout: { w: 12, h: 3 } + children: + - type: 'timeseries_stat' + title: 'Model calls' + icon: sparkles + query: &calls attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") | summarize count() by bin_auto(timestamp) + unit: calls + eager: true + layout: { w: 3, h: 2 } + + - type: 'timeseries_stat' + title: 'Cost' + icon: dollar + query: attributes.gen_ai.usage.cost != null | summarize sum(attributes.gen_ai.usage.cost) by bin_auto(timestamp) + unit: $ + eager: true + layout: { w: 3, h: 2 } + + - type: 'stat' + title: 'Model error rate' + icon: bug + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") | summarize countif(status_code == "ERROR") * 100.0 / count() + unit: '%' + eager: true + layout: { w: 3, h: 2 } + + - type: 'stat' + title: 'Conversations' + icon: comment + query: attributes.gen_ai.conversation.id != null | summarize dcount(attributes.gen_ai.conversation.id) + unit: conversations + eager: true + layout: { w: 3, h: 2 } + + - type: 'timeseries' + title: 'Cost by model' + query: attributes.gen_ai.usage.cost != null | summarize sum(attributes.gen_ai.usage.cost) by bin_auto(timestamp), attributes.gen_ai.request.model + unit: $ + layout: { w: 6, h: 4 } + + - type: 'timeseries' + title: 'Model calls by model' + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") | summarize count() by bin_auto(timestamp), attributes.gen_ai.request.model + unit: calls + layout: { w: 6, h: 4 } + + - type: 'timeseries' + title: 'Input tokens by model' + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") | summarize sum(attributes.gen_ai.usage.input_tokens) by bin_auto(timestamp), attributes.gen_ai.request.model + unit: tokens + layout: { w: 6, h: 4 } + + - type: 'timeseries' + title: 'Output tokens by model' + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") | summarize sum(attributes.gen_ai.usage.output_tokens) by bin_auto(timestamp), attributes.gen_ai.request.model + unit: tokens + layout: { w: 6, h: 4 } + + - type: 'timeseries_line' + title: 'Model latency percentiles' + hide_subtitle: true + summarize_by: max + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") and duration != null | summarize percentiles(duration / 1000000, 50, 95, 99) by bin_auto(timestamp) + unit: ms + layout: { w: 6, h: 4 } + + - type: 'timeseries' + title: 'Model errors by model' + theme: 'roma' + query: attributes.gen_ai.operation.name in ("chat", "text_completion", "generate_content", "embeddings") and status_code == "ERROR" | summarize count() by bin_auto(timestamp), attributes.gen_ai.request.model + unit: errors + layout: { w: 6, h: 4 } + + - type: 'timeseries' + title: 'Tool calls by tool' + query: attributes.gen_ai.operation.name == "execute_tool" | summarize count() by bin_auto(timestamp), attributes.gen_ai.tool.name + unit: calls + layout: { w: 6, h: 4 } + + - type: top_list + title: 'Failing tools' + query: attributes.gen_ai.operation.name == "execute_tool" and status_code == "ERROR" | summarize count() by attributes.gen_ai.tool.name | order by count_ desc + unit: errors + layout: { w: 3, h: 4 } + + - type: top_list + title: 'Cost by agent' + query: attributes.gen_ai.usage.cost != null and attributes.gen_ai.agent.name != null | summarize sum(attributes.gen_ai.usage.cost) by attributes.gen_ai.agent.name + unit: $ + layout: { w: 3, h: 4 } + + - type: 'timeseries' + title: 'Cost by service' + query: attributes.gen_ai.usage.cost != null | summarize sum(attributes.gen_ai.usage.cost) by bin_auto(timestamp), resource.service.name + unit: $ + layout: { w: 6, h: 4 } diff --git a/test/integration/Opentelemetry/GenAISpec.hs b/test/integration/Opentelemetry/GenAISpec.hs new file mode 100644 index 000000000..a3a18148a --- /dev/null +++ b/test/integration/Opentelemetry/GenAISpec.hs @@ -0,0 +1,111 @@ +-- | LLM telemetry end to end: every producer dialect lands as GenAI conventions with a cost, +-- and Monoscope's own assistant, traced and re-ingested, reads back as one agent run. +module Opentelemetry.GenAISpec (spec) where + +import Control.Lens ((^..), (^?)) +import Data.Aeson qualified as AE +import Data.Aeson.Lens (key, values, _Number, _String) +import Data.List (lookup) +import Data.Time (UTCTime, addUTCTime, getCurrentTime) +import Data.Time.Format.ISO8601 (iso8601Show) +import Data.UUID qualified as UUID +import Pages.BodyWrapper (PageCtx (..)) +import Pages.Bots.Utils qualified as Bot +import Pages.Components (Deferred (..)) +import Pages.LLM qualified as LLM +import Pages.LogExplorer.LogItem qualified as LogItem +import Pages.Telemetry qualified as TelemetryPage +import Pkg.AI qualified as AI +import Pkg.Components.Table (Table (..)) +import Pkg.TestUtils +import Proto.Opentelemetry.Proto.Logs.V1.Logs qualified as PL +import System.Config (AuthContext (..), EnvConfig (..)) + + +spec :: Spec +spec = around withTestResources do + describe "GenAI telemetry" do + it "maps OpenInference, OpenLLMetry and Vercel AI SDK spans onto the GenAI conventions and prices inference" \tr -> do + pid <- createTestProject tr "GenAI dialects" + apiKey <- createTestAPIKey tr pid "genai" + now <- getCurrentTime + let emitIn trId n name attrs = ingestSpanLinked tr apiKey trId ("a" <> show n) Nothing name attrs (addUTCTime (fromIntegral n) now) + emit = emitIn "6e1f" + emit (1 :: Int) "openinference llm" [("openinference.span.kind", "LLM"), ("llm.model_name", "gpt-4o"), ("llm.token_count.prompt", "1000"), ("llm.token_count.completion", "100"), ("llm.input_messages.0.message.role", "user"), ("llm.input_messages.0.message.content", "Why is checkout slow?"), ("session.id", "conv-1")] + emitIn "7a2b" 2 "openllmetry chat" [("gen_ai.system", "anthropic"), ("llm.request.type", "chat"), ("gen_ai.request.model", "claude-sonnet-4-5"), ("gen_ai.usage.prompt_tokens", "2000"), ("gen_ai.usage.completion_tokens", "50"), ("gen_ai.usage.cache_read_input_tokens", "1500"), ("gen_ai.completion.0.role", "assistant"), ("gen_ai.completion.0.content", "The pool is exhausted.")] + emit 3 "ai.generateText" [("ai.operationId", "ai.generateText"), ("ai.model.id", "gpt-4o-mini"), ("ai.usage.promptTokens", "300"), ("ai.telemetry.metadata.sessionId", "conv-1")] + emit 4 "ai.generateText.doGenerate" [("ai.operationId", "ai.generateText.doGenerate"), ("ai.model.id", "gpt-4o-mini"), ("ai.model.provider", "openai.chat"), ("ai.usage.promptTokens", "300"), ("ai.usage.completionTokens", "20"), ("ai.response.text", "Done."), ("ai.response.finishReason", "length"), ("ai.prompt.messages", "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Summarise\"}]}]")] + emit 5 "ai.toolCall" [("ai.operationId", "ai.toolCall"), ("ai.toolCall.name", "search_logs"), ("ai.toolCall.id", "call-9"), ("ai.toolCall.args", "{\"q\":\"timeout\"}")] + rows <- dbQuery @_ @(Text, AE.Value) tr [sql| SELECT name, attributes->'gen_ai' FROM otel_logs_and_spans WHERE project_id = ? ORDER BY name |] (Only pid) + let genAI name = fromMaybe AE.Null $ lookup name rows + path ks v = foldl' (\acc k -> acc >>= (^? key k)) (Just v) ks + [genAI n ^? key "operation" . key "name" . _String | n <- ["openinference llm", "openllmetry chat", "ai.generateText", "ai.generateText.doGenerate", "ai.toolCall"]] + `shouldBe` map Just ["chat", "chat", "invoke_agent", "chat", "execute_tool"] + -- gpt-4o: 1000 in at $2.50/M + 100 out at $10/M; sonnet: 500 uncached at $3, 1500 cached at $0.30, 50 out at $15. + [path ["usage", "cost"] (genAI n) ^? traverse . _Number | n <- ["openinference llm", "openllmetry chat", "ai.generateText"]] `shouldBe` [Just 0.0035, Just 0.002700, Nothing] + path ["usage", "cost_source"] (genAI "ai.generateText.doGenerate") `shouldBe` Just "estimated" + path ["conversation", "id"] (genAI "openinference llm") `shouldBe` Just "conv-1" + path ["conversation", "id"] (genAI "ai.generateText") `shouldBe` Just "conv-1" + path ["provider", "name"] (genAI "ai.generateText.doGenerate") `shouldBe` Just "openai" + genAI "openinference llm" ^.. key "input" . key "messages" . values . key "parts" . values . key "content" . _String `shouldBe` ["Why is checkout slow?"] + genAI "openllmetry chat" ^.. key "output" . key "messages" . values . key "parts" . values . key "content" . _String `shouldBe` ["The pool is exhausted."] + genAI "ai.generateText.doGenerate" ^.. key "input" . key "messages" . values . key "parts" . values . key "content" . _String `shouldBe` ["Summarise"] + map (`path` genAI "ai.toolCall") [["tool", "name"], ["tool", "call", "id"]] `shouldBe` [Just "search_logs", Just "call-9"] + -- Both conv-1 traces are one conversation; the lone OpenLLMetry call is only a call. + let (from, to) = around' now + LLM.LLMPage (PageCtx _ (DeferredBody (LLM.ConversationsBody table))) <- call tr $ LLM.llmGetH pid Nothing Nothing Nothing Nothing from to Nothing (Just "1") + map (\c -> (c.key, c.llmCalls, c.toolCalls, c.title, c.signals)) (toList table.rows) `shouldBe` [("conv-1", 2, 1, Just "Why is checkout slow?", [LLM.Truncated])] + calls <- call tr $ LLM.llmGetH pid (Just "calls") Nothing (Just "claude-sonnet-4-5") Nothing from to Nothing (Just "1") + renderT calls `shouldContainAll` ["openllmetry chat", "$0.0027", "2K → 50"] + -- The trace view prices each step and totals the trace: $0.0035 + $0.000057 for gpt-4o-mini. + [(trId, ts)] <- dbQuery tr [sql| SELECT context___trace_id, timestamp FROM otel_logs_and_spans WHERE project_id = ? AND name = 'openinference llm' |] (Only pid) + renderT <$> call tr (TelemetryPage.traceH pid trId (Just ts) Nothing Nothing Nothing Nothing) >>= (`shouldContainAll` ["1K → 100 tok · $0.0035", "LLM cost", "$0.0036"]) + + it "keeps each trace-less Claude Code session its own conversation, and leaves look-alike app logs alone" \tr -> do + pid <- createTestProject tr "Claude Code" + apiKey <- createTestAPIKey tr pid "claude-code" + now <- getCurrentTime + let event n session promptId attrs = ingestLogRecord tr apiKey ("INFO", PL.SEVERITY_NUMBER_INFO) ("claude_code." <> fromMaybe "" (lookup "event.name" attrs)) ([("session.id", session), ("prompt.id", promptId)] <> attrs) (addUTCTime n now) + event 1 "sess-a" "p1" [("event.name", "user_prompt"), ("prompt", "Fix the failing build")] + event 2 "sess-a" "p1" [("event.name", "api_request"), ("model", "claude-sonnet-4-5"), ("input_tokens", "10"), ("cache_read_tokens", "5000"), ("output_tokens", "20"), ("cost_usd", "0.0021")] + event 3 "sess-b" "p2" [("event.name", "user_prompt"), ("prompt", "Write the tests")] + event 4 "sess-b" "p2" [("event.name", "api_request"), ("model", "claude-sonnet-4-5"), ("input_tokens", "100"), ("output_tokens", "900"), ("cost_usd", "0.5")] + ingestLogRecord tr apiKey ("ERROR", PL.SEVERITY_NUMBER_ERROR) "checkout failed" [("event.name", "api_error"), ("session.id", "sess-a")] (addUTCTime 5 now) + let (from, to) = around' now + LLM.LLMPage (PageCtx _ (DeferredBody (LLM.ConversationsBody table))) <- call tr $ LLM.llmGetH pid Nothing Nothing Nothing Nothing from to Nothing (Just "1") + map (\c -> (c.key, c.title, c.llmCalls, c.tokens, c.cost)) (toList table.rows) `shouldBe` [("sess-b", Just "Write the tests", 1, 1000, Just 0.5), ("sess-a", Just "Fix the failing build", 1, 5030, Just 0.0021)] + renderT <$> call tr (LLM.conversationGetH pid "sess-a" from to Nothing) >>= (`shouldContainAll` ["Fix the failing build", "$0.0021"]) + + it "observes Monoscope's own assistant: an agent run, its model calls in order, and its tools" \tr -> do + apiKey <- createTestAPIKey tr testPid "self-telemetry" + answer <- withOwnTelemetry tr apiKey \traced -> + runTestBg frozenTime traced $ Bot.processAIQuery Nothing False AI.ServiceAccess testPid "plot error trend over time" Nothing tr.trATCtx.env.openaiModel tr.trATCtx.env.openaiApiKey + answer `shouldSatisfy` isRight + spans <- dbQuery @_ @(Text, AE.Value) tr [sql| SELECT name, attributes->'gen_ai' FROM otel_logs_and_spans WHERE project_id = ? AND attributes->'gen_ai' IS NOT NULL ORDER BY start_time |] (Only testPid) + let ops = map (\(_, g) -> g ^? key "operation" . key "name" . _String) spans + chats = [g | (_, g) <- spans, g ^? key "operation" . key "name" . _String == Just "chat"] + take 1 ops `shouldBe` [Just "invoke_agent"] + length chats `shouldSatisfy` (>= 2) + Just "execute_tool" `shouldSatisfy` (`elem` ops) + [n | (n, g) <- spans, g ^? key "agent" . key "name" . _String == Just "monoscope"] `shouldBe` ["invoke_agent monoscope"] + -- Each call carries the conversation so far, opening with the system prompt. + chats ^.. traverse . key "input" . key "messages" . values . key "role" . _String `shouldSatisfy` (\roles -> take 1 roles == ["system"] && "user" `elem` roles) + chats ^.. traverse . key "output" . key "messages" . values . key "role" . _String `shouldSatisfy` all (== "assistant") + -- The same run, read back through the pages a user would open. + -- Through TimeFusion reads, which CI points at a real TimeFusion. + (from, to) <- around' <$> getCurrentTime + let viaTf = tr{trATCtx = tr.trATCtx{env = tr.trATCtx.env{enableTimefusionReads = True}}} + LLM.LLMPage (PageCtx _ (DeferredBody (LLM.ConversationsBody table))) <- call viaTf $ LLM.llmGetH testPid Nothing Nothing Nothing Nothing from to Nothing (Just "1") + run <- maybe (fail "the assistant run is not listed as a conversation") pure $ viaNonEmpty head $ toList table.rows + (run.agents, run.title, run.llmCalls >= 2, run.toolCalls >= 1) `shouldBe` (["monoscope"], Just "plot error trend over time", True, True) + detail <- call viaTf $ LLM.conversationGetH testPid run.key from to Nothing + renderT detail `shouldContainAll` ["plot error trend over time", "System instructions", "Ask AI about this conversation", "Transcript", "Timeline", "execute_tool"] + [(chatId, chatTs)] <- dbQuery tr [sql| SELECT id::text, timestamp FROM otel_logs_and_spans WHERE project_id = ? AND attributes->'gen_ai'->'operation'->>'name' = 'chat' ORDER BY start_time LIMIT 1 |] (Only testPid) + chatUuid <- maybe (fail "span id is not a uuid") pure $ UUID.fromText chatId + drawer <- call tr $ LogItem.expandAPIlogItemH testPid chatUuid chatTs Nothing Nothing Nothing False + renderT drawer `shouldContainAll` ["tab-genai", "Conversation", "plot error trend over time"] + + +-- | An explicit window around the ingested spans, which carry real timestamps. +around' :: UTCTime -> (Maybe Text, Maybe Text) +around' t = (Just $ toText $ iso8601Show $ addUTCTime (-3600) t, Just $ toText $ iso8601Show $ addUTCTime 3600 t) diff --git a/web-components/src/log-list-utils.ts b/web-components/src/log-list-utils.ts index dbbb093d0..bd8f8d3d5 100644 --- a/web-components/src/log-list-utils.ts +++ b/web-components/src/log-list-utils.ts @@ -203,11 +203,11 @@ export const SUMMARY_ICON_CONFIGS = { iconClass: value === 'incoming' ? 'h-3 fill-iconNeutral' : 'h-3 fill-iconBrand', }), kind: (value: string) => - value === 'internal' + value === 'internal' || value === 'ai' ? { className: 'w-4 ml-2', - tooltip: 'Internal span', - iconName: 'function', + tooltip: value === 'ai' ? 'LLM / agent span' : 'Internal span', + iconName: value === 'ai' ? 'sparkles' : 'function', iconType: 'regular' as const, iconClass: 'h-3 w-3', }