feat(diagnostics): harden ingestion and delivery

2026-08-05 02:29:55 +02:00 · 2026-07-15 00:43:49 +02:00
parent ead4e09a60
commit b602a3acb6
42 changed files with 22428 additions and 141 deletions
--- a/services/diagnostics-api/test/apply-migrations.ts
+++ b/services/diagnostics-api/test/apply-migrations.ts
@@ -0,0 +1,4 @@
+import { env } from "cloudflare:workers";
+import { applyD1Migrations } from "cloudflare:test";
+
+await applyD1Migrations(env.DB, env.TEST_MIGRATIONS);
--- a/services/diagnostics-api/test/env.d.ts
+++ b/services/diagnostics-api/test/env.d.ts
@@ -0,0 +1,12 @@
+import type { D1Migration } from "@cloudflare/vitest-pool-workers";
+
+declare global {
+	namespace Cloudflare {
+		interface Env {
+			INGEST_KEY: string;
+			TEST_MIGRATIONS: D1Migration[];
+		}
+	}
+}
+
+export {};
--- a/services/diagnostics-api/test/input.test.ts
+++ b/services/diagnostics-api/test/input.test.ts
@@ -0,0 +1,266 @@
+import { describe, expect, it } from "vitest";
+import {
+	MAX_BREADCRUMBS_JSON_BYTES,
+	MAX_DEVICE_JSON_BYTES,
+	MAX_LOG_BYTES,
+	normalizeBug,
+	normalizeCrash,
+	normalizeEvents,
+	readJsonObject,
+} from "../src/input";
+
+const ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
+const INSTALL_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
+const ENCODER = new TextEncoder();
+
+describe("readJsonObject", () => {
+	it("enforces the byte limit on a chunked body without Content-Length", async () => {
+		const bytes = ENCODER.encode(JSON.stringify({ value: "😀".repeat(40) }));
+		const request = chunkedJsonRequest([
+			bytes.subarray(0, 7),
+			bytes.subarray(7, 31),
+			bytes.subarray(31),
+		]);
+
+		expect(request.headers.has("content-length")).toBe(false);
+		await expect(readJsonObject(request, 64)).resolves.toEqual({
+			ok: false,
+			status: 413,
+			error: "payload_too_large",
+		});
+	});
+
+	it("joins chunks before fatally decoding UTF-8", async () => {
+		const bytes = ENCODER.encode(JSON.stringify({ value: "😀" }));
+		const emojiStart = bytes.indexOf(0xf0);
+		const request = chunkedJsonRequest([
+			bytes.subarray(0, emojiStart + 2),
+			bytes.subarray(emojiStart + 2),
+		]);
+
+		await expect(readJsonObject(request, bytes.byteLength)).resolves.toEqual({
+			ok: true,
+			value: { value: "😀" },
+		});
+	});
+
+	it("rejects malformed UTF-8", async () => {
+		const request = chunkedJsonRequest([
+			new Uint8Array([0x7b, 0x22, 0x78, 0x22, 0x3a, 0x22, 0xc3, 0x28, 0x22, 0x7d]),
+		]);
+
+		await expect(readJsonObject(request, 100)).resolves.toEqual({
+			ok: false,
+			status: 400,
+			error: "invalid_utf8",
+		});
+	});
+
+	it("requires application/json with a UTF-8 charset", async () => {
+		const missing = new Request("https://example.test/v1/events", {
+			method: "POST",
+			body: "{}",
+		});
+		const wrongCharset = chunkedJsonRequest([ENCODER.encode("{}")], "application/json; charset=utf-16");
+
+		await expect(readJsonObject(missing, 100)).resolves.toEqual({
+			ok: false,
+			status: 415,
+			error: "unsupported_media_type",
+		});
+		await expect(readJsonObject(wrongCharset, 100)).resolves.toEqual({
+			ok: false,
+			status: 415,
+			error: "unsupported_media_type",
+		});
+	});
+
+	it("validates Content-Length before reading the body", async () => {
+		const invalid = chunkedJsonRequest([ENCODER.encode("{}")], "application/json", "invalid");
+		const oversized = chunkedJsonRequest(
+			[ENCODER.encode("{}")],
+			"application/json",
+			"999999999999999999999999999999999999",
+		);
+
+		await expect(readJsonObject(invalid, 100)).resolves.toEqual({
+			ok: false,
+			status: 400,
+			error: "invalid_content_length",
+		});
+		await expect(readJsonObject(oversized, 100)).resolves.toEqual({
+			ok: false,
+			status: 413,
+			error: "payload_too_large",
+		});
+	});
+
+	it("rejects a non-object JSON root", async () => {
+		const request = chunkedJsonRequest([ENCODER.encode("null")]);
+
+		await expect(readJsonObject(request, 100)).resolves.toEqual({
+			ok: false,
+			status: 400,
+			error: "invalid_body",
+		});
+	});
+});
+
+describe("normalizers", () => {
+	it("preserves false booleans and rejects their string representation", () => {
+		const crash = crashPayload(false);
+		const normalizedCrash = normalizeCrash(crash);
+		expect(normalizedCrash.ok).toBe(true);
+		if (normalizedCrash.ok) {
+			expect(normalizedCrash.value.diagnosticsEnabledAtCapture).toBe(false);
+		}
+		expect(normalizeCrash(crashPayload("false"))).toEqual({
+			ok: false,
+			status: 400,
+			error: "invalid_diagnostics_enabled",
+		});
+
+		const bug = bugPayload({ include_logs: false, logs: "discard me" });
+		const normalizedBug = normalizeBug(bug);
+		expect(normalizedBug.ok).toBe(true);
+		if (normalizedBug.ok) expect(normalizedBug.value.logs).toBe("");
+		const missingConsent = normalizeBug(bugPayload({ logs: "discard me" }));
+		expect(missingConsent.ok && missingConsent.value.logs).toBe("");
+		expect(normalizeBug(bugPayload({ include_logs: "false" }))).toEqual({
+			ok: false,
+			status: 400,
+			error: "invalid_include_logs",
+		});
+	});
+
+	it("keeps logs, breadcrumbs, and device JSON within valid byte budgets", () => {
+		const properties = Object.fromEntries(
+			Array.from({ length: 12 }, (_, index) => [`key-${index}-${"\u0000".repeat(40)}`, "\u0000".repeat(128)]),
+		);
+		const breadcrumbs = Array.from({ length: 40 }, (_, index) => ({
+			name: `crumb-${index}`,
+			timestamp_millis: index,
+			properties,
+		}));
+		const result = normalizeBug(
+			bugPayload({
+				include_logs: true,
+				logs: "😀".repeat(60_000),
+				breadcrumbs,
+				device: {
+					device_name: "\u0000".repeat(200),
+					device_model: "\u0000".repeat(200),
+					operating_system: "\u0000".repeat(300),
+					network: "\u0000".repeat(150),
+					battery_level: "\u0000".repeat(100),
+				},
+			}),
+		);
+
+		expect(result.ok).toBe(true);
+		if (!result.ok) return;
+		const breadcrumbsJson = JSON.stringify(result.value.breadcrumbs);
+		const deviceJson = JSON.stringify(result.value.device);
+		expect(ENCODER.encode(result.value.logs).byteLength).toBe(MAX_LOG_BYTES);
+		expect(ENCODER.encode(breadcrumbsJson).byteLength).toBeLessThanOrEqual(
+			MAX_BREADCRUMBS_JSON_BYTES,
+		);
+		expect(ENCODER.encode(deviceJson).byteLength).toBeLessThanOrEqual(MAX_DEVICE_JSON_BYTES);
+		expect(JSON.parse(breadcrumbsJson)).toEqual(result.value.breadcrumbs);
+		expect(JSON.parse(deviceJson)).toEqual(result.value.device);
+	});
+
+	it("requires stable report IDs and validates supplied IDs and schema versions", () => {
+		const result = normalizeEvents({
+			events: [{ name: "opened", ts: 1, schema_version: 1 }],
+		});
+		expect(result).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
+		const legacyInstall = normalizeEvents({
+			batch_id: ID,
+			install_id: "legacy-test-install",
+			events: [{ name: "opened", ts: 1 }],
+		});
+		expect(legacyInstall.ok && legacyInstall.value.installId).toBe("legacy-test-install");
+		const missingInstall = normalizeEvents({
+			batch_id: ID,
+			events: [{ name: "opened", ts: 1 }],
+		});
+		expect(missingInstall.ok && missingInstall.value.installId).toBe("unknown");
+		expect(
+			normalizeEvents({
+				batch_id: ID,
+				install_id: "bad\u0000install",
+				events: [{ name: "opened", ts: 1 }],
+			}),
+		).toEqual({ ok: false, status: 400, error: "invalid_install_id" });
+
+		expect(
+			normalizeEvents({
+				batch_id: "not-a-uuid",
+				events: [{ name: "opened", timestamp_millis: 1 }],
+			}),
+		).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
+		expect(
+			normalizeEvents({
+				batch_id: ID,
+				install_id: INSTALL_ID,
+				events: [{ name: "opened", timestamp_millis: 1, schema_version: 2 }],
+			}),
+		).toEqual({ ok: false, status: 400, error: "unsupported_schema_version" });
+	});
+});
+
+function chunkedJsonRequest(
+	chunks: readonly Uint8Array[],
+	contentType = "application/json; charset=utf-8",
+	contentLength?: string,
+): Request {
+	return new Request("https://example.test/v1/events", {
+		method: "POST",
+		headers: {
+			"content-type": contentType,
+			...(contentLength == null ? {} : { "content-length": contentLength }),
+		},
+		body: new ReadableStream<Uint8Array>({
+			start(controller) {
+				for (const chunk of chunks) controller.enqueue(chunk);
+				controller.close();
+			},
+		}),
+	});
+}
+
+function crashPayload(diagnosticsEnabled: unknown): Record<string, unknown> {
+	return {
+		id: ID,
+		install_id: INSTALL_ID,
+		app_version: "1.0",
+		platform: "test",
+		exception_type: "ExampleError",
+		exception_message: "message",
+		stack_trace: "stack",
+		occurred_at: 1,
+		diagnostics_enabled: diagnosticsEnabled,
+		schema_version: 1,
+		breadcrumbs: [],
+	};
+}
+
+function bugPayload(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+	return {
+		id: ID,
+		install_id: INSTALL_ID,
+		app_version: "1.0",
+		platform: "test",
+		occurred_at: 1,
+		what_happened: "It failed",
+		expected: "It worked",
+		steps: "Open the app",
+		contact: "",
+		logs: "",
+		device: {},
+		breadcrumbs: [],
+		schema_version: 1,
+		...overrides,
+	};
+}
--- a/services/diagnostics-api/test/tsconfig.json
+++ b/services/diagnostics-api/test/tsconfig.json
@@ -0,0 +1,7 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "types": ["@cloudflare/vitest-pool-workers/types", "node"]
+  },
+  "include": ["../worker-configuration.d.ts", "../vitest.config.ts", "./**/*.ts"]
+}
--- a/services/diagnostics-api/test/worker.test.ts
+++ b/services/diagnostics-api/test/worker.test.ts
@@ -0,0 +1,606 @@
+import { env, exports } from "cloudflare:workers";
+import { createExecutionContext } from "cloudflare:test";
+import { describe, expect, it, vi } from "vitest";
+import worker from "../src/index";
+import type {
+	NormalizedBugPayload,
+	NormalizedCrashPayload,
+	NormalizedEventsPayload,
+} from "../src/input";
+import {
+	type DiagnosticsEnv,
+	runRetention,
+	storeBug,
+	storeCrash,
+	storeEvents,
+} from "../src/storage";
+
+const INSTALL_ID = "10000000-0000-4000-8000-000000000000";
+
+describe("diagnostics Worker", () => {
+	it("keeps public routing narrow and fails closed", async () => {
+		const live = await exports.default.fetch(new Request("https://diagnostics.test/live"));
+		expect(live.status).toBe(200);
+		expect(await live.json()).toMatchObject({ ok: true, schema: 1 });
+
+		const health = await exports.default.fetch(healthRequest());
+		expect(health.status).toBe(200);
+		expect(await health.json()).toMatchObject({ ok: true });
+		const unauthorizedHealth = await exports.default.fetch(healthRequest("wrong-key"));
+		expect(unauthorizedHealth.status).toBe(401);
+
+		const unknown = await exports.default.fetch(
+			new Request("https://diagnostics.test/v1/not-real", { method: "POST" }),
+		);
+		expect(unknown.status).toBe(404);
+
+		const unauthorized = await exports.default.fetch(
+			jsonRequest("/v1/events", eventPayload(uuid(1)), "wrong-key"),
+		);
+		expect(unauthorized.status).toBe(401);
+		expect(await unauthorized.json()).toEqual({ error: "unauthorized" });
+		expect(unauthorized.headers.get("x-request-id")).toMatch(
+			/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/,
+		);
+
+		const preflight = await exports.default.fetch(
+			new Request("https://diagnostics.test/v1/events", { method: "OPTIONS" }),
+		);
+		expect(preflight.status).toBe(204);
+		expect(preflight.headers.get("access-control-allow-origin")).toBeNull();
+		expect(preflight.headers.get("allow")).toBe("POST, OPTIONS");
+	});
+
+	it("applies the source limit before shared-key verification", async () => {
+		let installLimitCalls = 0;
+		const limitedEnv: DiagnosticsEnv = {
+			...env,
+			SOURCE_RATE_LIMITER: {
+				limit: async () => ({ success: false }),
+			} as RateLimit,
+			INSTALL_RATE_LIMITER: {
+				limit: async () => {
+					installLimitCalls += 1;
+					return { success: true };
+				},
+			} as RateLimit,
+		};
+		const context = createExecutionContext();
+
+		const response = await worker.fetch(
+			jsonRequest("/v1/events", eventPayload(uuid(3)), "wrong-key", "198.51.100.3"),
+			limitedEnv,
+			context,
+		);
+
+		expect(response.status).toBe(429);
+		expect(response.headers.get("retry-after")).toBe("60");
+		expect(await response.json()).toEqual({ error: "rate_limited" });
+		expect(installLimitCalls).toBe(0);
+	});
+
+	it("returns structured errors for invalid bodies and asynchronous storage failures", async () => {
+		const invalid = await exports.default.fetch(jsonRequest("/v1/events", null));
+		expect(invalid.status).toBe(400);
+		expect(await invalid.json()).toEqual({ error: "invalid_body" });
+
+		const rejection = new Error("simulated D1 rejection");
+		const statement = {
+			bind: () => statement,
+			first: async () => Promise.reject(rejection),
+			run: async () => Promise.reject(rejection),
+		};
+		const rejectingDatabase = {
+			prepare: () => statement,
+		} as unknown as D1Database;
+		const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
+
+		const healthContext = createExecutionContext();
+		const unhealthy = await worker.fetch(
+			healthRequest(env.INGEST_KEY, "198.51.100.4"),
+			rejectingEnv,
+			healthContext,
+		);
+		expect(unhealthy.status).toBe(503);
+		expect(await unhealthy.json()).toEqual({ ok: false, error: "dependency_unavailable" });
+
+		const ingestContext = createExecutionContext();
+		const failedIngest = await worker.fetch(
+			jsonRequest("/v1/events", eventPayload(uuid(2)), env.INGEST_KEY, "198.51.100.2"),
+			rejectingEnv,
+			ingestContext,
+		);
+		expect(failedIngest.status).toBe(500);
+		expect(await failedIngest.json()).toEqual({ error: "internal" });
+	});
+
+	it("deduplicates event batches using the client batch ID", async () => {
+		const id = uuid(10);
+		const first = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
+		const second = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
+
+		expect(first.status).toBe(202);
+		expect(await first.json()).toMatchObject({
+			ok: true,
+			id,
+			stored: 1,
+			duplicate: false,
+		});
+		expect(second.status).toBe(202);
+		expect(await second.json()).toMatchObject({
+			ok: true,
+			id,
+			stored: 0,
+			duplicate: true,
+		});
+
+		const row = await env.DB.prepare(
+			"SELECT event_count AS eventCount, payload_json AS payloadJson FROM event_batches WHERE id = ?",
+		)
+			.bind(id)
+			.first<{ eventCount: number; payloadJson: string }>();
+		expect(row?.eventCount).toBe(1);
+		expect(JSON.parse(row?.payloadJson ?? "null")).toEqual([
+			{
+				name: "app_open",
+				timestampMillis: 1,
+				properties: { screen: "home" },
+				schemaVersion: 1,
+			},
+		]);
+	});
+
+	it("keeps D1 idempotency when the optional analytics index is enabled", async () => {
+		const points: AnalyticsEngineDataPoint[] = [];
+		const analytics = {
+			writeDataPoint: (point: AnalyticsEngineDataPoint) => points.push(point),
+		} as AnalyticsEngineDataset;
+		const analyticsEnv: DiagnosticsEnv = { ...env, AE: analytics };
+		const payload: NormalizedEventsPayload = {
+			batchId: uuid(11),
+			installId: INSTALL_ID,
+			appVersion: "1.0",
+			platform: "test",
+			events: [
+				{
+					name: "indexed",
+					timestampMillis: 1,
+					properties: {},
+					schemaVersion: 1,
+				},
+			],
+		};
+
+		expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: false, stored: 1 });
+		expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: true, stored: 0 });
+		expect(points).toHaveLength(1);
+	});
+
+	it("keeps the accepted crash blob when a duplicate request arrives", async () => {
+		const id = uuid(20);
+		const first = await exports.default.fetch(
+			jsonRequest("/v1/crashes", crashPayload(id, "first stack")),
+		);
+		const second = await exports.default.fetch(
+			jsonRequest("/v1/crashes", crashPayload(id, "second stack")),
+		);
+
+		expect(first.status).toBe(202);
+		const firstBody = await first.json<{ fingerprint: string }>();
+		expect(firstBody).toMatchObject({ ok: true, id, duplicate: false });
+		expect(second.status).toBe(202);
+		const secondBody = await second.json<{ fingerprint: string }>();
+		expect(secondBody).toMatchObject({ ok: true, id, duplicate: true });
+
+		const row = await env.DB.prepare(
+			`SELECT stack_r2_key AS stackKey, breadcrumbs_json AS breadcrumbsJson,
+			        fingerprint
+			 FROM crashes WHERE id = ?`,
+		)
+			.bind(id)
+			.first<{ stackKey: string; breadcrumbsJson: string; fingerprint: string }>();
+		expect(row?.stackKey).toMatch(new RegExp(`^crashes/${id}/[0-9a-f-]+/stack\\.txt$`));
+		expect(firstBody.fingerprint).toBe(row?.fingerprint);
+		expect(secondBody.fingerprint).toBe(row?.fingerprint);
+		expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([]);
+		expect(await (await env.BLOBS.get(row?.stackKey ?? "missing"))?.text()).toBe("first stack");
+
+		const objects = await env.BLOBS.list({ prefix: `crashes/${id}/` });
+		expect(objects.objects.map((object) => object.key)).toEqual([row?.stackKey]);
+	});
+
+	it("stores bug metadata as JSON and cleans the duplicate upload attempt", async () => {
+		const id = uuid(30);
+		const payload = bugPayload(id, "first logs");
+		const first = await exports.default.fetch(jsonRequest("/v1/bugs", payload));
+		const second = await exports.default.fetch(
+			jsonRequest("/v1/bugs", bugPayload(id, "second logs")),
+		);
+
+		expect(first.status).toBe(202);
+		expect(await first.json()).toMatchObject({ ok: true, id, duplicate: false });
+		expect(second.status).toBe(202);
+		expect(await second.json()).toMatchObject({ ok: true, id, duplicate: true });
+
+		const row = await env.DB.prepare(
+			`SELECT occurred_at AS occurredAt, logs_r2_key AS logsKey,
+			        device_json AS deviceJson, breadcrumbs_json AS breadcrumbsJson
+			 FROM bugs WHERE id = ?`,
+		)
+			.bind(id)
+			.first<{
+				occurredAt: number;
+				logsKey: string;
+				deviceJson: string;
+				breadcrumbsJson: string;
+			}>();
+		expect(row?.occurredAt).toBe(3);
+		expect(JSON.parse(row?.deviceJson ?? "null")).toEqual({
+			deviceName: "Test device",
+			deviceModel: "Model",
+			operatingSystem: "Test OS",
+			network: "offline",
+			batteryLevel: "90%",
+		});
+		expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([
+			{ name: "opened", timestampMillis: 2, properties: {} },
+		]);
+		expect(await (await env.BLOBS.get(row?.logsKey ?? "missing"))?.text()).toBe("first logs");
+
+		const objects = await env.BLOBS.list({ prefix: `bugs/${id}/` });
+		expect(objects.objects.map((object) => object.key)).toEqual([row?.logsKey]);
+	});
+
+	it("acknowledges known report IDs without touching an unavailable blob store", async () => {
+		const crash = normalizedCrash(uuid(31), "accepted stack");
+		const bug = normalizedBug(uuid(32), "accepted logs");
+		const firstCrash = await storeCrash(crash, env);
+		await storeBug(bug, env);
+		let blobWrites = 0;
+		const unavailableBlobs = {
+			put: async () => {
+				blobWrites += 1;
+				throw new Error("simulated R2 outage");
+			},
+		} as unknown as R2Bucket;
+		const unavailableEnv: DiagnosticsEnv = { ...env, BLOBS: unavailableBlobs };
+
+		await expect(
+			storeCrash({ ...crash, stackTrace: "retry stack" }, unavailableEnv),
+		).resolves.toEqual({
+			id: crash.id,
+			duplicate: true,
+			stored: 0,
+			fingerprint: firstCrash.fingerprint,
+		});
+		await expect(
+			storeBug({ ...bug, logs: "retry logs" }, unavailableEnv),
+		).resolves.toEqual({ id: bug.id, duplicate: true, stored: 0 });
+		expect(blobWrites).toBe(0);
+	});
+
+	it("removes uploaded report blobs when D1 rejects the metadata write", async () => {
+		const rejection = new Error("simulated D1 write rejection");
+		const rejectingDatabase = databaseWithSession(
+			() => null,
+			async () => Promise.reject(rejection),
+		);
+		const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
+		const crashId = uuid(33);
+		const bugId = uuid(34);
+
+		const crashResponse = await worker.fetch(
+			jsonRequest("/v1/crashes", crashPayload(crashId, "orphan candidate"), env.INGEST_KEY, "198.51.100.33"),
+			rejectingEnv,
+			createExecutionContext(),
+		);
+		const bugResponse = await worker.fetch(
+			jsonRequest("/v1/bugs", bugPayload(bugId, "orphan candidate"), env.INGEST_KEY, "198.51.100.34"),
+			rejectingEnv,
+			createExecutionContext(),
+		);
+
+		expect(crashResponse.status).toBe(500);
+		expect(await crashResponse.json()).toEqual({ error: "internal" });
+		expect(bugResponse.status).toBe(500);
+		expect(await bugResponse.json()).toEqual({ error: "internal" });
+		expect((await env.BLOBS.list({ prefix: `crashes/${crashId}/` })).objects).toEqual([]);
+		expect((await env.BLOBS.list({ prefix: `bugs/${bugId}/` })).objects).toEqual([]);
+	});
+
+	it("removes expired rows and their exact R2 objects while preserving current data", async () => {
+		const oldEventId = uuid(40);
+		const oldCrashId = uuid(41);
+		const oldBugId = uuid(42);
+		const currentEventId = uuid(43);
+		const oldCrashKey = `crashes/${oldCrashId}/retention/stack.txt`;
+		const oldBugKey = `bugs/${oldBugId}/retention/logs.txt`;
+		const oldReceivedAt = Date.now() - 100 * 86_400_000;
+
+		await Promise.all([
+			env.BLOBS.put(oldCrashKey, "expired crash"),
+			env.BLOBS.put(oldBugKey, "expired logs"),
+		]);
+		await env.DB.batch([
+			env.DB.prepare(
+				`INSERT INTO event_batches
+				 (id, received_at, install_id, app_version, platform, event_count, payload_json)
+				 VALUES (?, ?, ?, '', '', 1, '[]')`,
+			).bind(oldEventId, oldReceivedAt, INSTALL_ID),
+			env.DB.prepare(
+				`INSERT INTO event_batches
+				 (id, received_at, install_id, app_version, platform, event_count, payload_json)
+				 VALUES (?, ?, ?, '', '', 1, '[]')`,
+			).bind(currentEventId, Date.now(), INSTALL_ID),
+			env.DB.prepare(
+				`INSERT INTO crashes
+				 (id, received_at, occurred_at, install_id, app_version, platform,
+				  exception_type, exception_message, fingerprint, diagnostics_enabled,
+				  stack_r2_key, breadcrumbs_json, schema_version)
+				 VALUES (?, ?, ?, ?, '', '', 'Error', '', 'fingerprint', 1, ?, '[]', 1)`,
+			).bind(oldCrashId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldCrashKey),
+			env.DB.prepare(
+				`INSERT INTO bugs
+				 (id, received_at, occurred_at, install_id, app_version, platform,
+				  what_happened, expected, steps, contact, logs_r2_key,
+				  device_json, breadcrumbs_json, status, schema_version)
+				 VALUES (?, ?, ?, ?, '', '', 'failed', 'worked', '', '', ?, '{}', '[]', 'open', 1)`,
+			).bind(oldBugId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldBugKey),
+		]);
+
+		await runRetention(env);
+
+		for (const [table, id] of [
+			["event_batches", oldEventId],
+			["crashes", oldCrashId],
+			["bugs", oldBugId],
+		] as const) {
+			const row = await env.DB.prepare(`SELECT id FROM ${table} WHERE id = ?`).bind(id).first();
+			expect(row).toBeNull();
+		}
+		expect(await env.BLOBS.head(oldCrashKey)).toBeNull();
+		expect(await env.BLOBS.head(oldBugKey)).toBeNull();
+		expect(
+			await env.DB.prepare("SELECT id FROM event_batches WHERE id = ?").bind(currentEventId).first(),
+		).not.toBeNull();
+	});
+
+	it("bounds a full retention run below the D1 per-invocation query limit", async () => {
+		let queryCount = 0;
+		let batchCalls = 0;
+		const blobDeleteBatchSizes: number[] = [];
+		const rows = Array.from({ length: 900 }, (_, index) => ({
+			id: `expired-${index}`,
+			blobKey: `expired/${index}`,
+		}));
+		const database = {
+			prepare: () => {
+				const statement = {
+					bind: () => statement,
+					all: async () => {
+						queryCount += 1;
+						return d1Result(rows, 0);
+					},
+				};
+				return statement;
+			},
+			batch: async (statements: D1PreparedStatement[]) => {
+				batchCalls += 1;
+				queryCount += statements.length;
+				if (batchCalls === 9) {
+					return statements.map(() => d1Result([{ count: 1 }], 0));
+				}
+				return statements.map((_, index) => d1Result([], index === 0 ? 1_000 : 900));
+			},
+		} as unknown as D1Database;
+		const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+		const blobs = {
+			delete: async (keys: string | string[]) => {
+				blobDeleteBatchSizes.push(typeof keys === "string" ? 1 : keys.length);
+			},
+		} as unknown as R2Bucket;
+
+		try {
+			await runRetention({ ...env, DB: database, BLOBS: blobs });
+		} finally {
+			warning.mockRestore();
+		}
+
+		expect(queryCount).toBe(43);
+		expect(blobDeleteBatchSizes).toHaveLength(16);
+		expect(Math.max(...blobDeleteBatchSizes)).toBe(1_000);
+	});
+
+	it("converges an expired report backlog across bounded retention runs", async () => {
+		const oldReceivedAt = Date.now() - 100 * 86_400_000;
+		await env.DB.prepare(
+			`WITH digits(value) AS (
+			   VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9)
+			 ), sequence(value) AS (
+			   SELECT thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value + 1
+			   FROM digits AS thousands
+			   CROSS JOIN digits AS hundreds
+			   CROSS JOIN digits AS tens
+			   CROSS JOIN digits AS ones
+			   WHERE thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value < 7201
+			 )
+			 INSERT INTO crashes (
+			   id, received_at, occurred_at, install_id, app_version, platform,
+			   exception_type, exception_message, fingerprint, diagnostics_enabled,
+			   stack_r2_key, breadcrumbs_json, schema_version
+			 )
+			 SELECT 'retention-backlog-' || printf('%04d', value), ?, ?, ?, '', '',
+			        'Error', '', 'fingerprint-' || value, 0, NULL, '[]', 1
+			 FROM sequence`,
+		)
+			.bind(oldReceivedAt, oldReceivedAt, INSTALL_ID)
+			.run();
+		const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+
+		try {
+			await runRetention(env);
+			const afterFirstRun = await env.DB.prepare(
+				"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
+			).first<{ count: number }>();
+			expect(afterFirstRun?.count).toBe(1);
+
+			await runRetention(env);
+			const afterSecondRun = await env.DB.prepare(
+				"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
+			).first<{ count: number }>();
+			expect(afterSecondRun?.count).toBe(0);
+		} finally {
+			warning.mockRestore();
+		}
+	});
+});
+
+function jsonRequest(
+	path: string,
+	body: unknown,
+	key = env.INGEST_KEY,
+	source = "198.51.100.1",
+): Request {
+	return new Request(`https://diagnostics.test${path}`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json; charset=utf-8",
+			"cf-connecting-ip": source,
+			"x-vnidrop-install-id": INSTALL_ID,
+			"x-vnidrop-key": key,
+		},
+		body: JSON.stringify(body),
+	});
+}
+
+function healthRequest(key = env.INGEST_KEY, source = "198.51.100.1"): Request {
+	return new Request("https://diagnostics.test/health", {
+		headers: {
+			"cf-connecting-ip": source,
+			"x-vnidrop-key": key,
+		},
+	});
+}
+
+function eventPayload(batchId: string): Record<string, unknown> {
+	return {
+		batchId,
+		installId: INSTALL_ID,
+		appVersion: "1.0",
+		platform: "test",
+		events: [
+			{
+				name: "app_open",
+				timestampMillis: 1,
+				properties: { screen: "home" },
+				schemaVersion: 1,
+			},
+		],
+	};
+}
+
+function crashPayload(id: string, stackTrace: string): Record<string, unknown> {
+	return {
+		id,
+		installId: INSTALL_ID,
+		appVersion: "1.0",
+		platform: "test",
+		exceptionType: "TestError",
+		exceptionMessage: "failed",
+		stackTrace,
+		timestampMillis: 2,
+		diagnosticsEnabledAtCapture: true,
+		breadcrumbs: [],
+		schemaVersion: 1,
+	};
+}
+
+function bugPayload(id: string, logs: string): Record<string, unknown> {
+	return {
+		id,
+		installId: INSTALL_ID,
+		appVersion: "1.0",
+		platform: "test",
+		timestampMillis: 3,
+		whatHappened: "It failed",
+		expected: "It should work",
+		steps: "Open the app",
+		contact: "",
+		includeLogs: true,
+		logs,
+		device: {
+			deviceName: "Test device",
+			deviceModel: "Model",
+			operatingSystem: "Test OS",
+			network: "offline",
+			batteryLevel: "90%",
+		},
+		breadcrumbs: [{ name: "opened", timestampMillis: 2, properties: {} }],
+		schemaVersion: 1,
+	};
+}
+
+function normalizedCrash(id: string, stackTrace: string): NormalizedCrashPayload {
+	return {
+		id,
+		installId: INSTALL_ID,
+		appVersion: "1.0",
+		platform: "test",
+		exceptionType: "TestError",
+		exceptionMessage: "failed",
+		stackTrace,
+		occurredAt: 2,
+		diagnosticsEnabledAtCapture: true,
+		breadcrumbs: [],
+		schemaVersion: 1,
+	};
+}
+
+function normalizedBug(id: string, logs: string): NormalizedBugPayload {
+	return {
+		id,
+		installId: INSTALL_ID,
+		appVersion: "1.0",
+		platform: "test",
+		occurredAt: 3,
+		whatHappened: "It failed",
+		expected: "It should work",
+		steps: "Open the app",
+		contact: "",
+		logs,
+		device: {
+			deviceName: "Test device",
+			deviceModel: "Model",
+			operatingSystem: "Test OS",
+			network: "offline",
+			batteryLevel: "90%",
+		},
+		breadcrumbs: [],
+		schemaVersion: 1,
+	};
+}
+
+function databaseWithSession(
+	first: () => unknown,
+	run: () => Promise<D1Result>,
+): D1Database {
+	const statement = {
+		bind: () => statement,
+		first: async () => first(),
+		run,
+	} as unknown as D1PreparedStatement;
+	const session = {
+		prepare: () => statement,
+	} as unknown as D1DatabaseSession;
+	return {
+		withSession: () => session,
+	} as unknown as D1Database;
+}
+
+function d1Result<T>(results: T[], changes: number): D1Result<T> {
+	return { success: true, results, meta: { changes } } as D1Result<T>;
+}
+
+function uuid(suffix: number): string {
+	return `00000000-0000-4000-8000-${suffix.toString().padStart(12, "0")}`;
+}