{
  "meta": {
    "version": "v1",
    "pricing": {
      "public": {
        "label": "Public",
        "description": "Read-only API access for lightweight status checks and public integrations."
      },
      "premium": {
        "label": "Premium",
        "description": "API keys with higher hourly quotas, plus Slack, Teams, Discord, webhook, and email outage alerts across your vendor stack."
      }
    },
    "generatedAt": "2026-11-18T01:00:11.463Z"
  },
  "data": {
    "id": "incident_statuspage_buildkite_3bjtdp9tll09",
    "slug": "buildkite-increased-latency-and-error-rates-2025-10-20",
    "title": "Increased latency and error rates",
    "summary": "Increased latency and error rates",
    "status": "resolved",
    "stale": false,
    "severity": "major",
    "startedAt": "2025-10-20T14:18:27.869+00:00",
    "updatedAt": "2025-10-24T21:30:21.399+00:00",
    "resolvedAt": "2025-10-20T22:02:41.493+00:00",
    "provider": {
      "slug": "buildkite",
      "name": "Buildkite"
    },
    "affectedServices": [
      {
        "slug": "buildkite-pipelines",
        "name": "Pipelines & job queue"
      },
      {
        "slug": "buildkite-api",
        "name": "REST API"
      }
    ],
    "links": {
      "html": "/incidents/buildkite-increased-latency-and-error-rates-2025-10-20",
      "api": "/api/v1/incidents/buildkite-increased-latency-and-error-rates-2025-10-20",
      "providerHtml": "/providers/buildkite",
      "alerts": "https://outagedeck.com/account?stack=buildkite&utm_source=api&utm_medium=response&utm_campaign=api_alerts&utm_content=incident"
    },
    "impactSummary": "Buildkite reported a major event for the affected tracked services.",
    "source": {
      "id": "source_buildkite_status",
      "kind": "official_api",
      "name": "Buildkite Status",
      "checkedAt": "2026-11-18T00:50:11.599+00:00",
      "stale": false,
      "officialUrl": "https://www.buildkitestatus.com",
      "statusPageUrl": "https://www.buildkitestatus.com"
    },
    "updates": [
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_dcf13j78pbyf",
        "status": "investigating",
        "body": "We're observing increased latency and error rates due to an inability to scale up. We're currently investigating and will provide status updates as they become available.",
        "createdAt": "2025-10-20T14:18:27.923+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_x3l24rszg4d7",
        "status": "investigating",
        "body": "We're currently working on mitigations for scaling up, but at this stage service is degraded with increased latency across API, notifications, and builds starting.",
        "createdAt": "2025-10-20T15:17:44.233+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_qy3nx3m34wxx",
        "status": "identified",
        "body": "We're continuing to see increased latency across much of our sub-systems due to an on going AWS outage. We are unable to launch new tasks in us-east-1 and are investigating potential mitigations to restore service.",
        "createdAt": "2025-10-20T17:04:53.264+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_7qrcwyt2nhcf",
        "status": "identified",
        "body": "We have implemented mitigations and see an improvement in latency for the Agent API. Latency and error rates continue to be elevated across Rest, GraphQL and Web service as well as notifications being delayed.\n\nWe are continuing to work through mitigations and will provide an update in 1 hour.",
        "createdAt": "2025-10-20T17:36:56.223+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_rk7rz8l3ttfx",
        "status": "identified",
        "body": "Our mitigations improved latency for the Agent API, although latency and error rates are still visible across other services. The us-east-1 issue is reporting some recovery and we are seeing further improvements in our services. We are actively monitoring the situation and implementing mitigations where possible.",
        "createdAt": "2025-10-20T18:54:32.987+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_09h1ff64b4vf",
        "status": "monitoring",
        "body": "We're seeing slow recovery of all our services. Latency and error rates are decreasing across the board. We are continuing to monitor the situation.",
        "createdAt": "2025-10-20T19:29:19.037+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_2jgsq7b5gbyc",
        "status": "monitoring",
        "body": "We're seeing signs of recovery across the board. Error rates have reduced to baseline levels. Latency is trending towards baseline.\n\nWe continue to actively monitor our services and the AWS reports on us-east-1 impact.",
        "createdAt": "2025-10-20T20:33:09.5+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_fty64spt8yvx",
        "status": "monitoring",
        "body": "Latency and error rates have all returned to baseline levels. We have seen full recovery of our services.\n\nWe continue to actively monitor our services and the AWS reports on us-east-1 impact to ensure stability is maintained.",
        "createdAt": "2025-10-20T21:04:58.022+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_3cbw0prv6qc7",
        "status": "resolved",
        "body": "Our services have been fully recovered for the last hour, so we are marking this as resolved. \n\nOur engineers will continue to monitor AWS and will keep services scaled up to prevent impact from any additional failures.",
        "createdAt": "2025-10-20T22:02:41.493+00:00"
      },
      {
        "id": "update_statuspage_buildkite_3bjtdp9tll09_1ngxt559z8jt",
        "status": "resolved",
        "body": "We have published a post-incident review here:\n\n[https://buildkite.com/resources/blog/post-incident-review-for-20th-october-2025/](https://buildkite.com/resources/blog/post-incident-review-for-20th-october-2025/)",
        "createdAt": "2025-10-24T20:22:57.468+00:00"
      }
    ],
    "access": {
      "plan": "public",
      "keyed": false
    }
  }
}