refactor(diagnostics-api): drop telemetry and crash ingestion, keep bug reports

Remove the /v1/events and /v1/crashes routes, their normalizers and storage
paths, and simplify retention to the bugs table. Add a migration dropping the
now-unused event_batches and crashes tables, and regenerate worker types.
This commit is contained in:
2026-08-02 10:03:12 +02:00
parent b68d338097
commit 3e18378610
9 changed files with 122 additions and 730 deletions

View File

@@ -4,8 +4,6 @@ import {
MAX_DEVICE_JSON_BYTES,
MAX_LOG_BYTES,
normalizeBug,
normalizeCrash,
normalizeEvents,
readJsonObject,
} from "../src/input";
@@ -57,7 +55,7 @@ describe("readJsonObject", () => {
});
it("requires application/json with a UTF-8 charset", async () => {
const missing = new Request("https://example.test/v1/events", {
const missing = new Request("https://example.test/v1/bugs", {
method: "POST",
body: "{}",
});
@@ -108,18 +106,6 @@ describe("readJsonObject", () => {
describe("normalizers", () => {
it("preserves false booleans and rejects their string representation", () => {
const crash = crashPayload(false);
const normalizedCrash = normalizeCrash(crash);
expect(normalizedCrash.ok).toBe(true);
if (normalizedCrash.ok) {
expect(normalizedCrash.value.diagnosticsEnabledAtCapture).toBe(false);
}
expect(normalizeCrash(crashPayload("false"))).toEqual({
ok: false,
status: 400,
error: "invalid_diagnostics_enabled",
});
const bug = bugPayload({ include_logs: false, logs: "discard me" });
const normalizedBug = normalizeBug(bug);
expect(normalizedBug.ok).toBe(true);
@@ -171,42 +157,31 @@ describe("normalizers", () => {
});
it("requires stable report IDs and validates supplied IDs and schema versions", () => {
const result = normalizeEvents({
events: [{ name: "opened", ts: 1, schema_version: 1 }],
});
expect(result).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
const legacyInstall = normalizeEvents({
batch_id: ID,
install_id: "legacy-test-install",
events: [{ name: "opened", ts: 1 }],
});
expect(legacyInstall.ok && legacyInstall.value.installId).toBe("legacy-test-install");
const missingInstall = normalizeEvents({
batch_id: ID,
events: [{ name: "opened", ts: 1 }],
});
expect(missingInstall.ok && missingInstall.value.installId).toBe("unknown");
expect(
normalizeEvents({
batch_id: ID,
install_id: "bad\u0000install",
events: [{ name: "opened", ts: 1 }],
}),
).toEqual({ ok: false, status: 400, error: "invalid_install_id" });
const missingId = normalizeBug(bugPayload({ id: undefined }));
expect(missingId).toEqual({ ok: false, status: 400, error: "invalid_id" });
expect(
normalizeEvents({
batch_id: "not-a-uuid",
events: [{ name: "opened", timestamp_millis: 1 }],
}),
).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
expect(
normalizeEvents({
batch_id: ID,
install_id: INSTALL_ID,
events: [{ name: "opened", timestamp_millis: 1, schema_version: 2 }],
}),
).toEqual({ ok: false, status: 400, error: "unsupported_schema_version" });
const legacyInstall = normalizeBug(bugPayload({ install_id: "legacy-test-install" }));
expect(legacyInstall.ok && legacyInstall.value.installId).toBe("legacy-test-install");
const missingInstall = normalizeBug(bugPayload({ install_id: undefined }));
expect(missingInstall.ok && missingInstall.value.installId).toBe("unknown");
expect(normalizeBug(bugPayload({ install_id: "bad\u0000install" }))).toEqual({
ok: false,
status: 400,
error: "invalid_install_id",
});
expect(normalizeBug(bugPayload({ id: "not-a-uuid" }))).toEqual({
ok: false,
status: 400,
error: "invalid_id",
});
expect(normalizeBug(bugPayload({ schema_version: 2 }))).toEqual({
ok: false,
status: 400,
error: "unsupported_schema_version",
});
});
});
@@ -215,7 +190,7 @@ function chunkedJsonRequest(
contentType = "application/json; charset=utf-8",
contentLength?: string,
): Request {
return new Request("https://example.test/v1/events", {
return new Request("https://example.test/v1/bugs", {
method: "POST",
headers: {
"content-type": contentType,
@@ -230,24 +205,8 @@ function chunkedJsonRequest(
});
}
function crashPayload(diagnosticsEnabled: unknown): Record<string, unknown> {
return {
id: ID,
install_id: INSTALL_ID,
app_version: "1.0",
platform: "test",
exception_type: "ExampleError",
exception_message: "message",
stack_trace: "stack",
occurred_at: 1,
diagnostics_enabled: diagnosticsEnabled,
schema_version: 1,
breadcrumbs: [],
};
}
function bugPayload(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
const payload: Record<string, unknown> = {
id: ID,
install_id: INSTALL_ID,
app_version: "1.0",
@@ -263,4 +222,9 @@ function bugPayload(overrides: Record<string, unknown> = {}): Record<string, unk
schema_version: 1,
...overrides,
};
// An explicit `undefined` override omits the key entirely (simulating a missing field).
for (const key of Object.keys(overrides)) {
if (overrides[key] === undefined) delete payload[key];
}
return payload;
}

View File

@@ -2,18 +2,8 @@ import { env, exports } from "cloudflare:workers";
import { createExecutionContext } from "cloudflare:test";
import { describe, expect, it, vi } from "vitest";
import worker from "../src/index";
import type {
NormalizedBugPayload,
NormalizedCrashPayload,
NormalizedEventsPayload,
} from "../src/input";
import {
type DiagnosticsEnv,
runRetention,
storeBug,
storeCrash,
storeEvents,
} from "../src/storage";
import type { NormalizedBugPayload } from "../src/input";
import { type DiagnosticsEnv, runRetention, storeBug } from "../src/storage";
const INSTALL_ID = "10000000-0000-4000-8000-000000000000";
@@ -35,7 +25,7 @@ describe("diagnostics Worker", () => {
expect(unknown.status).toBe(404);
const unauthorized = await exports.default.fetch(
jsonRequest("/v1/events", eventPayload(uuid(1)), "wrong-key"),
jsonRequest("/v1/bugs", bugPayload(uuid(1), "logs"), "wrong-key"),
);
expect(unauthorized.status).toBe(401);
expect(await unauthorized.json()).toEqual({ error: "unauthorized" });
@@ -44,7 +34,7 @@ describe("diagnostics Worker", () => {
);
const preflight = await exports.default.fetch(
new Request("https://diagnostics.test/v1/events", { method: "OPTIONS" }),
new Request("https://diagnostics.test/v1/bugs", { method: "OPTIONS" }),
);
expect(preflight.status).toBe(204);
expect(preflight.headers.get("access-control-allow-origin")).toBeNull();
@@ -68,7 +58,7 @@ describe("diagnostics Worker", () => {
const context = createExecutionContext();
const response = await worker.fetch(
jsonRequest("/v1/events", eventPayload(uuid(3)), "wrong-key", "198.51.100.3"),
jsonRequest("/v1/bugs", bugPayload(uuid(3), "logs"), "wrong-key", "198.51.100.3"),
limitedEnv,
context,
);
@@ -80,7 +70,7 @@ describe("diagnostics Worker", () => {
});
it("returns structured errors for invalid bodies and asynchronous storage failures", async () => {
const invalid = await exports.default.fetch(jsonRequest("/v1/events", null));
const invalid = await exports.default.fetch(jsonRequest("/v1/bugs", null));
expect(invalid.status).toBe(400);
expect(await invalid.json()).toEqual({ error: "invalid_body" });
@@ -92,6 +82,8 @@ describe("diagnostics Worker", () => {
};
const rejectingDatabase = {
prepare: () => statement,
batch: async () => Promise.reject(rejection),
withSession: () => ({ prepare: () => statement }),
} as unknown as D1Database;
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
@@ -106,7 +98,7 @@ describe("diagnostics Worker", () => {
const ingestContext = createExecutionContext();
const failedIngest = await worker.fetch(
jsonRequest("/v1/events", eventPayload(uuid(2)), env.INGEST_KEY, "198.51.100.2"),
jsonRequest("/v1/bugs", bugPayload(uuid(2), "logs"), env.INGEST_KEY, "198.51.100.2"),
rejectingEnv,
ingestContext,
);
@@ -114,101 +106,6 @@ describe("diagnostics Worker", () => {
expect(await failedIngest.json()).toEqual({ error: "internal" });
});
it("deduplicates event batches using the client batch ID", async () => {
const id = uuid(10);
const first = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
const second = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
expect(first.status).toBe(202);
expect(await first.json()).toMatchObject({
ok: true,
id,
stored: 1,
duplicate: false,
});
expect(second.status).toBe(202);
expect(await second.json()).toMatchObject({
ok: true,
id,
stored: 0,
duplicate: true,
});
const row = await env.DB.prepare(
"SELECT event_count AS eventCount, payload_json AS payloadJson FROM event_batches WHERE id = ?",
)
.bind(id)
.first<{ eventCount: number; payloadJson: string }>();
expect(row?.eventCount).toBe(1);
expect(JSON.parse(row?.payloadJson ?? "null")).toEqual([
{
name: "app_open",
timestampMillis: 1,
properties: { screen: "home" },
schemaVersion: 1,
},
]);
});
it("keeps D1 idempotency when the optional analytics index is enabled", async () => {
const points: AnalyticsEngineDataPoint[] = [];
const analytics = {
writeDataPoint: (point: AnalyticsEngineDataPoint) => points.push(point),
} as AnalyticsEngineDataset;
const analyticsEnv: DiagnosticsEnv = { ...env, AE: analytics };
const payload: NormalizedEventsPayload = {
batchId: uuid(11),
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
events: [
{
name: "indexed",
timestampMillis: 1,
properties: {},
schemaVersion: 1,
},
],
};
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: false, stored: 1 });
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: true, stored: 0 });
expect(points).toHaveLength(1);
});
it("keeps the accepted crash blob when a duplicate request arrives", async () => {
const id = uuid(20);
const first = await exports.default.fetch(
jsonRequest("/v1/crashes", crashPayload(id, "first stack")),
);
const second = await exports.default.fetch(
jsonRequest("/v1/crashes", crashPayload(id, "second stack")),
);
expect(first.status).toBe(202);
const firstBody = await first.json<{ fingerprint: string }>();
expect(firstBody).toMatchObject({ ok: true, id, duplicate: false });
expect(second.status).toBe(202);
const secondBody = await second.json<{ fingerprint: string }>();
expect(secondBody).toMatchObject({ ok: true, id, duplicate: true });
const row = await env.DB.prepare(
`SELECT stack_r2_key AS stackKey, breadcrumbs_json AS breadcrumbsJson,
fingerprint
FROM crashes WHERE id = ?`,
)
.bind(id)
.first<{ stackKey: string; breadcrumbsJson: string; fingerprint: string }>();
expect(row?.stackKey).toMatch(new RegExp(`^crashes/${id}/[0-9a-f-]+/stack\\.txt$`));
expect(firstBody.fingerprint).toBe(row?.fingerprint);
expect(secondBody.fingerprint).toBe(row?.fingerprint);
expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([]);
expect(await (await env.BLOBS.get(row?.stackKey ?? "missing"))?.text()).toBe("first stack");
const objects = await env.BLOBS.list({ prefix: `crashes/${id}/` });
expect(objects.objects.map((object) => object.key)).toEqual([row?.stackKey]);
});
it("stores bug metadata as JSON and cleans the duplicate upload attempt", async () => {
const id = uuid(30);
const payload = bugPayload(id, "first logs");
@@ -252,9 +149,7 @@ describe("diagnostics Worker", () => {
});
it("acknowledges known report IDs without touching an unavailable blob store", async () => {
const crash = normalizedCrash(uuid(31), "accepted stack");
const bug = normalizedBug(uuid(32), "accepted logs");
const firstCrash = await storeCrash(crash, env);
await storeBug(bug, env);
let blobWrites = 0;
const unavailableBlobs = {
@@ -265,14 +160,6 @@ describe("diagnostics Worker", () => {
} as unknown as R2Bucket;
const unavailableEnv: DiagnosticsEnv = { ...env, BLOBS: unavailableBlobs };
await expect(
storeCrash({ ...crash, stackTrace: "retry stack" }, unavailableEnv),
).resolves.toEqual({
id: crash.id,
duplicate: true,
stored: 0,
fingerprint: firstCrash.fingerprint,
});
await expect(
storeBug({ ...bug, logs: "retry logs" }, unavailableEnv),
).resolves.toEqual({ id: bug.id, duplicate: true, stored: 0 });
@@ -286,59 +173,27 @@ describe("diagnostics Worker", () => {
async () => Promise.reject(rejection),
);
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
const crashId = uuid(33);
const bugId = uuid(34);
const crashResponse = await worker.fetch(
jsonRequest("/v1/crashes", crashPayload(crashId, "orphan candidate"), env.INGEST_KEY, "198.51.100.33"),
rejectingEnv,
createExecutionContext(),
);
const bugResponse = await worker.fetch(
jsonRequest("/v1/bugs", bugPayload(bugId, "orphan candidate"), env.INGEST_KEY, "198.51.100.34"),
rejectingEnv,
createExecutionContext(),
);
expect(crashResponse.status).toBe(500);
expect(await crashResponse.json()).toEqual({ error: "internal" });
expect(bugResponse.status).toBe(500);
expect(await bugResponse.json()).toEqual({ error: "internal" });
expect((await env.BLOBS.list({ prefix: `crashes/${crashId}/` })).objects).toEqual([]);
expect((await env.BLOBS.list({ prefix: `bugs/${bugId}/` })).objects).toEqual([]);
});
it("removes expired rows and their exact R2 objects while preserving current data", async () => {
const oldEventId = uuid(40);
const oldCrashId = uuid(41);
const oldBugId = uuid(42);
const currentEventId = uuid(43);
const oldCrashKey = `crashes/${oldCrashId}/retention/stack.txt`;
const currentBugId = uuid(43);
const oldBugKey = `bugs/${oldBugId}/retention/logs.txt`;
const oldReceivedAt = Date.now() - 100 * 86_400_000;
await Promise.all([
env.BLOBS.put(oldCrashKey, "expired crash"),
env.BLOBS.put(oldBugKey, "expired logs"),
]);
await env.BLOBS.put(oldBugKey, "expired logs");
await env.DB.batch([
env.DB.prepare(
`INSERT INTO event_batches
(id, received_at, install_id, app_version, platform, event_count, payload_json)
VALUES (?, ?, ?, '', '', 1, '[]')`,
).bind(oldEventId, oldReceivedAt, INSTALL_ID),
env.DB.prepare(
`INSERT INTO event_batches
(id, received_at, install_id, app_version, platform, event_count, payload_json)
VALUES (?, ?, ?, '', '', 1, '[]')`,
).bind(currentEventId, Date.now(), INSTALL_ID),
env.DB.prepare(
`INSERT INTO crashes
(id, received_at, occurred_at, install_id, app_version, platform,
exception_type, exception_message, fingerprint, diagnostics_enabled,
stack_r2_key, breadcrumbs_json, schema_version)
VALUES (?, ?, ?, ?, '', '', 'Error', '', 'fingerprint', 1, ?, '[]', 1)`,
).bind(oldCrashId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldCrashKey),
env.DB.prepare(
`INSERT INTO bugs
(id, received_at, occurred_at, install_id, app_version, platform,
@@ -346,28 +201,28 @@ describe("diagnostics Worker", () => {
device_json, breadcrumbs_json, status, schema_version)
VALUES (?, ?, ?, ?, '', '', 'failed', 'worked', '', '', ?, '{}', '[]', 'open', 1)`,
).bind(oldBugId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldBugKey),
env.DB.prepare(
`INSERT INTO bugs
(id, received_at, occurred_at, install_id, app_version, platform,
what_happened, expected, steps, contact, logs_r2_key,
device_json, breadcrumbs_json, status, schema_version)
VALUES (?, ?, ?, ?, '', '', 'failed', 'worked', '', '', NULL, '{}', '[]', 'open', 1)`,
).bind(currentBugId, Date.now(), Date.now(), INSTALL_ID),
]);
await runRetention(env);
for (const [table, id] of [
["event_batches", oldEventId],
["crashes", oldCrashId],
["bugs", oldBugId],
] as const) {
const row = await env.DB.prepare(`SELECT id FROM ${table} WHERE id = ?`).bind(id).first();
expect(row).toBeNull();
}
expect(await env.BLOBS.head(oldCrashKey)).toBeNull();
expect(
await env.DB.prepare("SELECT id FROM bugs WHERE id = ?").bind(oldBugId).first(),
).toBeNull();
expect(await env.BLOBS.head(oldBugKey)).toBeNull();
expect(
await env.DB.prepare("SELECT id FROM event_batches WHERE id = ?").bind(currentEventId).first(),
await env.DB.prepare("SELECT id FROM bugs WHERE id = ?").bind(currentBugId).first(),
).not.toBeNull();
});
it("bounds a full retention run below the D1 per-invocation query limit", async () => {
let queryCount = 0;
let batchCalls = 0;
const blobDeleteBatchSizes: number[] = [];
const rows = Array.from({ length: 900 }, (_, index) => ({
id: `expired-${index}`,
@@ -381,17 +236,17 @@ describe("diagnostics Worker", () => {
queryCount += 1;
return d1Result(rows, 0);
},
run: async () => {
queryCount += 1;
return d1Result([], rows.length);
},
first: async () => {
queryCount += 1;
return { count: rows.length };
},
};
return statement;
},
batch: async (statements: D1PreparedStatement[]) => {
batchCalls += 1;
queryCount += statements.length;
if (batchCalls === 9) {
return statements.map(() => d1Result([{ count: 1 }], 0));
}
return statements.map((_, index) => d1Result([], index === 0 ? 1_000 : 900));
},
} as unknown as D1Database;
const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
const blobs = {
@@ -406,9 +261,10 @@ describe("diagnostics Worker", () => {
warning.mockRestore();
}
expect(queryCount).toBe(43);
expect(blobDeleteBatchSizes).toHaveLength(16);
expect(Math.max(...blobDeleteBatchSizes)).toBe(1_000);
// Eight passes (one SELECT + one DELETE each) plus the final backlog SELECT.
expect(queryCount).toBe(17);
expect(blobDeleteBatchSizes).toHaveLength(8);
expect(Math.max(...blobDeleteBatchSizes)).toBe(900);
});
it("converges an expired report backlog across bounded retention runs", async () => {
@@ -424,13 +280,13 @@ describe("diagnostics Worker", () => {
CROSS JOIN digits AS ones
WHERE thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value < 7201
)
INSERT INTO crashes (
INSERT INTO bugs (
id, received_at, occurred_at, install_id, app_version, platform,
exception_type, exception_message, fingerprint, diagnostics_enabled,
stack_r2_key, breadcrumbs_json, schema_version
what_happened, expected, steps, contact, logs_r2_key,
device_json, breadcrumbs_json, status, schema_version
)
SELECT 'retention-backlog-' || printf('%04d', value), ?, ?, ?, '', '',
'Error', '', 'fingerprint-' || value, 0, NULL, '[]', 1
'failed', 'worked', '', '', NULL, '{}', '[]', 'open', 1
FROM sequence`,
)
.bind(oldReceivedAt, oldReceivedAt, INSTALL_ID)
@@ -440,13 +296,13 @@ describe("diagnostics Worker", () => {
try {
await runRetention(env);
const afterFirstRun = await env.DB.prepare(
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
"SELECT COUNT(*) AS count FROM bugs WHERE id LIKE 'retention-backlog-%'",
).first<{ count: number }>();
expect(afterFirstRun?.count).toBe(1);
await runRetention(env);
const afterSecondRun = await env.DB.prepare(
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
"SELECT COUNT(*) AS count FROM bugs WHERE id LIKE 'retention-backlog-%'",
).first<{ count: number }>();
expect(afterSecondRun?.count).toBe(0);
} finally {
@@ -482,39 +338,6 @@ function healthRequest(key = env.INGEST_KEY, source = "198.51.100.1"): Request {
});
}
function eventPayload(batchId: string): Record<string, unknown> {
return {
batchId,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
events: [
{
name: "app_open",
timestampMillis: 1,
properties: { screen: "home" },
schemaVersion: 1,
},
],
};
}
function crashPayload(id: string, stackTrace: string): Record<string, unknown> {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
exceptionType: "TestError",
exceptionMessage: "failed",
stackTrace,
timestampMillis: 2,
diagnosticsEnabledAtCapture: true,
breadcrumbs: [],
schemaVersion: 1,
};
}
function bugPayload(id: string, logs: string): Record<string, unknown> {
return {
id,
@@ -540,22 +363,6 @@ function bugPayload(id: string, logs: string): Record<string, unknown> {
};
}
function normalizedCrash(id: string, stackTrace: string): NormalizedCrashPayload {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
exceptionType: "TestError",
exceptionMessage: "failed",
stackTrace,
occurredAt: 2,
diagnosticsEnabledAtCapture: true,
breadcrumbs: [],
schemaVersion: 1,
};
}
function normalizedBug(id: string, logs: string): NormalizedBugPayload {
return {
id,