{ "meta": { "version": "v1", "pricing": { "public": { "label": "Public", "description": "Read-only API access for lightweight status checks and public integrations." }, "premium": { "label": "Premium", "description": "API keys with higher hourly quotas, plus Slack, Teams, Discord, webhook, and email outage alerts across your vendor stack." } }, "generatedAt": "2026-10-03T20:39:50.387Z" }, "data": { "id": "incident_statuspage_harness_17bslwl25dhx", "slug": "harness-data-ingestion-is-delayed-on-traceable-us-production-2026-08-19", "title": "Data ingestion is delayed on Traceable US production", "summary": "Data ingestion is delayed on Traceable US production", "status": "resolved", "stale": false, "severity": "major", "startedAt": "2026-08-19T13:22:48.966+00:00", "updatedAt": "2026-08-27T23:35:38.537+00:00", "resolvedAt": "2026-08-19T16:07:13.081+00:00", "provider": { "slug": "harness", "name": "Harness" }, "affectedServices": [], "links": { "html": "/incidents/harness-data-ingestion-is-delayed-on-traceable-us-production-2026-08-19", "api": "/api/v1/incidents/harness-data-ingestion-is-delayed-on-traceable-us-production-2026-08-19", "providerHtml": "/providers/harness", "alerts": "https://outagedeck.com/account?stack=harness&utm_source=api&utm_medium=response&utm_campaign=api_alerts&utm_content=incident" }, "impactSummary": "Harness reported a major event for the affected tracked services.", "source": { "id": "source_harness_status", "kind": "official_api", "name": "Harness Status", "checkedAt": "2026-10-03T20:35:04.322+00:00", "stale": false, "officialUrl": "https://status.harness.io", "statusPageUrl": "https://status.harness.io" }, "updates": [ { "id": "update_statuspage_harness_17bslwl25dhx_098qcvyj4bp9", "status": "investigating", "body": "We are currently investigating this issue.", "createdAt": "2026-08-19T13:22:49.056+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_6xztnqfts9yg", "status": "investigating", "body": "We are continuing to investigate this issue.", "createdAt": "2026-08-19T14:03:59.947+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_bwmmh1ytn15x", "status": "investigating", "body": "We are continuing to investigate this issue.", "createdAt": "2026-08-19T14:55:14.809+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_dhc0d75r9bv3", "status": "identified", "body": "The issue has been identified and a fix is being implemented.", "createdAt": "2026-08-19T15:13:46.285+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_lfzt9kkplk5g", "status": "monitoring", "body": "A fix has been implemented and we are monitoring the results.", "createdAt": "2026-08-19T15:22:30.691+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_fnz0rjpclwpp", "status": "monitoring", "body": "We are continuing to monitor for any further issues.", "createdAt": "2026-08-19T15:33:26.254+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_150bmhkz0pqk", "status": "resolved", "body": "This incident has been resolved.", "createdAt": "2026-08-19T16:07:13.081+00:00" }, { "id": "update_statuspage_harness_17bslwl25dhx_15brqp9c3z0d", "status": "resolved", "body": "**Summary**\n\nOn 19 August 2026 between 12:35 and 17:29 UTC, the Harness Application Security service experienced a significant disruption affecting both the customer-facing console and the data ingestion pipeline in the SaaS Production and US1 regions.\n\n‌\n\n**Root Cause** \n\nThe internal configuration service that supplies runtime settings to nearly every other component became overloaded and entered a repeated restart cycle. Because so many services depend on it, the effects were broad: console pages such as protection policies, posture views, activity logs, API inventory, and custom policy failed to load or timed out, and downstream processing stalled while waiting for configuration it could not obtain.\n\n# **Customer impact**\n\n| **Dimension** | **Detail** |\n| --- | --- |\n| Console \\(UI\\) impact | Multiple pages failed to load or timed out, including protection policies, posture event pages and posture views inside dashboards and insight pages, activity log queries, API inventory screens, custom policy, and sensitive-data views and widgets. |\n| Ingestion impact | Security telemetry processing degraded severely and, in some paths, stopped entirely. Consumer lag grew across normalisation, grouping, anomaly detection, generation, and related processing stages. |\n| Data loss | A subset of telemetry ingested during the disruption was permanently dropped. |\n\n‌\n\n**Mitigation**\n\n Several intermediate mitigations additional CPU and memory, relaxed health-check thresholds, a database restart, and a larger connection pool ameliorated the issue. Disabling the new feature in both affected regions restored throughput sharply and durably. The incident was resolved at 17:29 UTC.\n\n‌\n\n# **Preventive actions**\n\nThe following actions are committed and tracked internally to completion. The feature that triggered this incident remains disabled and will not be re-enabled until the work below is complete and validated.\n\n| **Action** |\n| --- |\n| |\n| OPtimize the code by tuning parameters such as cache eviction and retention , evaluate cursor-based pagination for bulk rule retrieval as rule counts grow |\n| Add a purpose-built database index for the service-scoping access pattern |\n| Remediate pipeline recovery semantics so consumers replay safely after position-marker loss instead of skipping backlog |\n| Mandate staged rollout for configuration overrides that alter downstream request patterns: low-volume cluster, then mid-volume, then high-volume |\n| Add backpressure and concurrency protection to the configuration service: circuit breaking, bounded queues, and timeout isolation |\n| Enhance observability by Instrumenting more detailed metrics |", "createdAt": "2026-08-26T17:20:52.266+00:00" } ], "access": { "plan": "public", "keyed": false } } }