{
  "schemaVersion": 1,
  "id": "gateway-overhead-linux-2026-10-18",
  "metric": "Gateway overhead",
  "unit": "ms",
  "percentile": "p99",
  "statistic": "Mean of eight block p99 estimates",
  "publishedOn": "2026-10-18",
  "axisStep": 4,
  "source": {
    "name": "Aixy controlled Linux benchmark",
    "url": "/benchmarks",
    "sourceSha": "85f1c06ec718f19cb78969e42e5b1878062c43b9",
    "productionSha": "6b356b45f4dccd9a13a514dcb7075d0c3ac9590f",
    "summarySha256": "2d2e8af1410a302bae69f07cf6d0611cae3724ebe7778ed8bf0cc024f4fa955d",
    "campaignSha256": "d6233735dee7caf6c82910ce76ceb7b4b131142b459bec5a3f21fc51e86f5998",
    "manifestSha256": "7bd7feb6de11bade7206d5988eaf3d54c6d3d9cc9cb88003617a534fc44445d0"
  },
  "campaign": {
    "blocks": 8,
    "cells": 96,
    "measuredRequests": 13506726,
    "includesDirectControls": true,
    "minRequestsPerCell": 50000,
    "minDurationSeconds": 60,
    "warmupRequests": 300,
    "concurrency": 1,
    "promptBytes": 1024,
    "maxOutputTokens": 256,
    "upstreamDelayMs": 0,
    "protocol": "OpenAI-compatible JSON Chat Completions",
    "host": "Frankfurt c7g.2xlarge, Ubuntu 24.04 arm64",
    "gatewayCores": 1,
    "gatewayMemoryGiB": 4,
    "gatewaySwapGiB": 0,
    "order": "Eight balanced Williams blocks; fresh gateway per target, with direct controls before and after.",
    "metricDefinition": "Each client response duration minus its matching provider service duration. This includes the client and loopback baseline.",
    "aggregation": "Mean of eight block quantiles, not a pooled percentile.",
    "intervalDefinition": "Two-sided 95% Student-t interval over eight block p99 estimates (df=7), assuming approximately independent normal block effects on one host."
  },
  "results": [
    {
      "id": "aixy",
      "name": "Aixy",
      "version": "85f1c06e",
      "value": 0.476125,
      "percentile": "p99",
      "basis": "measured",
      "interval95": {
        "lower": 0.4737004363712183,
        "upper": 0.4785495636287817
      },
      "quantiles": {
        "p50": 0.432,
        "p95": 0.45325,
        "p99": 0.476125,
        "p999": 0.689125,
        "maxMean": 4.9955,
        "maxObserved": 8.688
      },
      "blockP99Ms": [
        0.47,
        0.474,
        0.477,
        0.479,
        0.478,
        0.478,
        0.476,
        0.477
      ],
      "clientP99Ms": 0.512,
      "directBeforeP99Ms": 0.327375,
      "directAfterP99Ms": 0.326875,
      "quantileContrastMs": 0.184875,
      "responseWindowCpuUsPerCompletion": 193.9625,
      "profile": "Native Rust production forwarding, compiled authorization and pricing, usage projection, OTLP encoding and commercial metering with bounded local acknowledgements."
    },
    {
      "id": "bifrost",
      "name": "Bifrost",
      "version": "2.3.0",
      "value": 2.14875,
      "percentile": "p99",
      "basis": "measured",
      "interval95": {
        "lower": 2.1304623409206878,
        "upper": 2.167037659079312
      },
      "quantiles": {
        "p50": 0.72775,
        "p95": 0.956875,
        "p99": 2.14875,
        "p999": 4.636,
        "maxMean": 11.42925,
        "maxObserved": 14.885
      },
      "blockP99Ms": [
        2.114,
        2.137,
        2.172,
        2.152,
        2.138,
        2.144,
        2.185,
        2.148
      ],
      "clientP99Ms": 2.1865,
      "directBeforeP99Ms": 0.32725,
      "directAfterP99Ms": 0.327625,
      "quantileContrastMs": 1.8590625,
      "responseWindowCpuUsPerCompletion": 610.8875,
      "profile": "Official Go gateway with virtual-key authorization and a fresh SQLite configuration store; request logging disabled."
    },
    {
      "id": "portkey",
      "name": "Portkey OSS",
      "version": "1.15.2",
      "value": 7.41675,
      "percentile": "p99",
      "basis": "measured",
      "interval95": {
        "lower": 7.38635038604951,
        "upper": 7.44714961395049
      },
      "quantiles": {
        "p50": 1.2495,
        "p95": 1.439125,
        "p99": 7.41675,
        "p999": 18.232,
        "maxMean": 35.403875,
        "maxObserved": 37.618
      },
      "blockP99Ms": [
        7.432,
        7.407,
        7.359,
        7.438,
        7.444,
        7.47,
        7.379,
        7.405
      ],
      "clientP99Ms": 7.45975,
      "directBeforeP99Ms": 0.327125,
      "directAfterP99Ms": 0.32725,
      "quantileContrastMs": 7.1325625,
      "responseWindowCpuUsPerCompletion": 1295.8375,
      "profile": "Official Node.js 20.19.6 application and dependencies, provider-key forwarding and built-in usage handling; no separate caller-authorization layer."
    },
    {
      "id": "litellm",
      "name": "LiteLLM (Python)",
      "version": "1.105.1",
      "value": 15.550625,
      "percentile": "p99",
      "basis": "measured",
      "interval95": {
        "lower": 15.49306355665404,
        "upper": 15.608186443345959
      },
      "quantiles": {
        "p50": 14.19825,
        "p95": 15.140875,
        "p99": 15.550625,
        "p999": 15.964625,
        "maxMean": 229.29825,
        "maxObserved": 231.911
      },
      "blockP99Ms": [
        15.568,
        15.564,
        15.417,
        15.558,
        15.595,
        15.655,
        15.54,
        15.508
      ],
      "clientP99Ms": 15.6085,
      "directBeforeP99Ms": 0.327625,
      "directAfterP99Ms": 0.327125,
      "quantileContrastMs": 15.281125,
      "responseWindowCpuUsPerCompletion": 14188.5625,
      "profile": "Standard Python/Uvicorn proxy with a static master key and normal usage/cost processing; no database, configured callbacks or Rust inference."
    }
  ],
  "limitations": [
    "Synthetic, closed-loop JSON traffic at concurrency one; no production capacity, real-model latency or streaming performance claim.",
    "The native Aixy fixture exercises production forwarding and usage components, not the complete deployed service.",
    "Configured authorization and accounting features differ between products. Local acknowledgements do not establish remote durable delivery.",
    "Bifrost and Portkey percentage contrasts are descriptive. p99.9 and maxima are point diagnostics, not confidence or worst-case bounds.",
    "This campaign measures standard Python LiteLLM, not its opt-in Rust inference gateway."
  ],
  "pricingSensitivity": {
    "order": [
      "A",
      "B",
      "B",
      "A"
    ],
    "requestsPerCell": 2000,
    "measuredRequests": 24000,
    "cells": 12,
    "finalEligible": false,
    "unpricedP99Ms": [
      15.392,
      15.473
    ],
    "explicitTariffP99Ms": [
      16.378,
      16.397
    ],
    "description": "A separate post-final diagnostic added only explicit positive synthetic-model prices to LiteLLM. Pricing did not accelerate these runs. The diagnostic does not replace any main result or supply final performance confidence."
  }
}