feat(diagnostics): harden ingestion and delivery

This commit is contained in:
2026-07-15 00:43:49 +02:00
parent ead4e09a60
commit b602a3acb6
42 changed files with 22428 additions and 141 deletions

53
.github/workflows/diagnostics-api.yml vendored Normal file
View File

@@ -0,0 +1,53 @@
name: Diagnostics API
on:
pull_request:
paths:
- "services/diagnostics-api/**"
- ".github/workflows/diagnostics-api.yml"
push:
branches:
- master
paths:
- "services/diagnostics-api/**"
- ".github/workflows/diagnostics-api.yml"
permissions:
contents: read
concurrency:
group: diagnostics-api-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
quality:
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: services/diagnostics-api
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
cache-dependency-path: services/diagnostics-api/package-lock.json
- name: Install dependencies
run: npm ci
- name: Verify generated Worker types
run: npm run types:check
- name: Type-check
run: npm run typecheck
- name: Test in the Workers runtime
run: npm test
- name: Validate the deployment bundle
run: npm run deploy:dry-run

View File

@@ -18,3 +18,8 @@ android.useAndroidX=true
# Bug report UI remains available (user-initiated). # Bug report UI remains available (user-initiated).
# Override per build: ./gradlew … -Pvnidrop.diagnostics.included=false # Override per build: ./gradlew … -Pvnidrop.diagnostics.included=false
vnidrop.diagnostics.included=true vnidrop.diagnostics.included=true
# Cloudflare Worker base URL (no trailing slash). Both endpoint/key empty → NoOp transport.
# Example: https://vnidrop-diagnostics.<subdomain>.workers.dev
vnidrop.diagnostics.endpoint=
# Shared ingest key (must match Worker secret INGEST_KEY). Keep blank in VCS; use user-level/CI properties.
vnidrop.diagnostics.ingestKey=

6
services/diagnostics-api/.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
node_modules/
.wrangler/
dist/
.dev.vars*
.env*
*.log

View File

@@ -0,0 +1,196 @@
# VniDrop diagnostics API
Cloudflare Worker for ingesting batched telemetry, crash reports, and user-submitted
bug reports. D1 stores searchable metadata; R2 stores larger stack traces and logs.
The service is designed for modest traffic and low operating cost:
- one D1 row is written per telemetry batch, not per event;
- crash stacks and bug logs are stored in R2 instead of D1;
- request and batch limits reject oversized work before storage writes;
- an hourly scheduled cleanup and an R2 lifecycle rule enforce retention;
- no Queue, Durable Object, or KV resources are required.
Cloudflare quotas and prices change over time. Check the current
[Workers pricing](https://developers.cloudflare.com/workers/platform/pricing/),
[D1 pricing](https://developers.cloudflare.com/d1/platform/pricing/), and
[R2 pricing](https://developers.cloudflare.com/r2/pricing/) before relying on a
particular free-plan capacity.
## API
All ingest routes require:
```http
X-VniDrop-Key: <INGEST_KEY>
```
The client also sends its anonymous installation ID:
```http
X-VniDrop-Install-Id: <anonymous install UUID>
```
| Method | Path | Body |
|--------|------|------|
| `GET` | `/live` | process liveness; does not touch storage |
| `GET` | `/health` | authenticated readiness; checks required configuration and the D1 schema |
| `POST` | `/v1/events` | `{ batchId, installId, appVersion?, platform?, events: [...] }` |
| `POST` | `/v1/crashes` | app crash payload |
| `POST` | `/v1/bugs` | app bug-report payload |
Batch and report IDs are client-generated UUIDs. A client must reuse the same ID
when retrying so D1 can acknowledge the request without storing it twice.
Accepted reports return `202`. Defaults are a 262,144-byte request limit and at
most 50 events per batch. Cloudflare rate-limit bindings allow 30 requests per
installation and 120 requests per source, per ingest route, per minute. Source
limits run before shared-key verification so rejected traffic is bounded too.
These counters are eventually consistent and local to a Cloudflare location, so
they are abuse mitigation rather than billing or authorization controls.
The API is consumed by native Android, iOS, and desktop clients and does not
enable cross-origin browser access. If a browser-based client is added later,
define a narrow origin allowlist instead of enabling wildcard CORS.
`/health` requires `X-VniDrop-Key` and uses the source limiter because it performs
D1 reads. `/live` is the only unauthenticated probe and never touches storage.
## Security model
`INGEST_KEY` fails closed when it is missing, but it is a shared value embedded in
released app binaries. It can be extracted and therefore is **not** user
authentication, a durable secret, or sufficient abuse protection by itself.
- Store the Worker value with `wrangler secret put`; never put it in
`wrangler.jsonc`, source control, logs, or command arguments.
- Rotate the key when it is exposed and ship the matching app configuration.
- Keep the two rate-limit namespaces unique within the Cloudflare account. A
namespace reused by another Worker shares counters with it.
- Use Cloudflare WAF or account-level rate-limiting rules if public abuse exceeds
what the Worker bindings can absorb.
- Do not log request bodies. Bug reports can contain contact details and attached
logs.
## Provision and deploy
Run these commands from this directory:
```bash
npm ci
npx wrangler login
npx wrangler d1 create vnidrop-diagnostics
npx wrangler r2 bucket create vnidrop-diagnostics
```
Replace the placeholder `database_id` in `wrangler.jsonc` with the UUID returned
by `wrangler d1 create`. Set the ingest key interactively, apply the tracked D1
migrations, and configure the R2 retention rule once:
```bash
npx wrangler secret put INGEST_KEY
npm run db:migrate:remote
npx wrangler r2 bucket lifecycle add vnidrop-diagnostics diagnostics-retention --expire-days 90
npm run check
npm run deploy
```
`npm run deploy` also runs the complete `check` script automatically before
Wrangler changes the remote Worker.
The lifecycle command changes the remote bucket. Before adding or changing a
rule, inspect the current state with:
```bash
npx wrangler r2 bucket lifecycle list vnidrop-diagnostics
```
## Local development
Create an ignored `.dev.vars` file containing a development-only key:
```dotenv
INGEST_KEY=local-development-only
```
Then initialize the local D1 database and run the Worker:
```bash
npm run db:migrate:local
npm run dev
```
Wrangler keeps local D1 and R2 state under the ignored `.wrangler/` directory.
Use `wrangler dev --test-scheduled` when exercising the hourly cleanup handler.
## Migrations and generated types
D1 migrations live in `migrations/` and are recorded in D1's migration ledger.
Never edit an applied migration; add the next numbered SQL file instead.
`worker-configuration.d.ts` is generated from `wrangler.jsonc` and committed so
bindings cannot silently drift from the Worker code:
```bash
npm run typegen # regenerate after changing bindings or vars
npm run types:check # verify the committed file is current
```
Secrets and optional, commented-out bindings are not generated. The source adds
only those narrow extensions to the generated environment type.
Vitest runs inside the Workers runtime. Its setup applies the same numbered D1
migrations to the isolated local database assigned to each test file.
## Retention
`RETENTION_DAYS` defaults to 90. The `17 * * * *` cron trigger runs cleanup at
17 minutes past every hour. Cleanup works in bounded batches: it deletes each
expired report's referenced R2 object before deleting that exact D1 row. The R2
lifecycle rule is an independent backstop for stack and log objects, including
objects left behind by a partial ingest failure. Each scheduled run can remove
8,000 event batches and 7,200 rows from each report table while staying below
D1's per-invocation query ceiling. Later hourly runs continue any backlog.
Reaching the cap emits a structured warning with the remaining expired-row counts;
alert on that warning because
retention is necessarily best-effort during sustained distributed abuse.
The Worker variable and bucket lifecycle are separate configuration surfaces.
When changing retention, update both `RETENTION_DAYS` and the R2 lifecycle rule;
changing one does not update the other. Cloudflare may delete expired R2 objects
after the exact expiration time rather than synchronously at it.
## App wiring
Keep the tracked root defaults empty. Configure release builds through the
user-level `~/.gradle/gradle.properties` or secured CI Gradle project properties:
```properties
vnidrop.diagnostics.included=true
vnidrop.diagnostics.endpoint=https://vnidrop-diagnostics.<your-subdomain>.workers.dev
vnidrop.diagnostics.ingestKey=<same value as INGEST_KEY>
```
Both the endpoint and key are required. When both are empty the app uses its
offline-safe no-op transport; configuring only one fails the Gradle build.
`vnidrop.diagnostics.included=false` disables
automatic telemetry and crash upload, but a configured endpoint can still accept
an explicit user-submitted bug report. Treat the app-side key as an abuse-control
token with the limitations described above.
## Reading reports
```bash
npx wrangler d1 execute vnidrop-diagnostics --remote \
--command "SELECT id, exception_type, platform, occurred_at FROM crashes ORDER BY occurred_at DESC LIMIT 20"
npx wrangler d1 execute vnidrop-diagnostics --remote \
--command "SELECT id, what_happened, status, occurred_at FROM bugs WHERE status = 'open' ORDER BY occurred_at DESC LIMIT 20"
```
R2 object keys use `crashes/<id>/<attempt-id>/stack.txt` and
`bugs/<id>/<attempt-id>/logs.txt`. The unique attempt segment prevents a retry
from overwriting an already accepted object before D1 detects the duplicate.
There is no public administration endpoint; inspect reports through authenticated
Cloudflare tools or a future Access-protected dashboard.

View File

@@ -0,0 +1,55 @@
-- This migration also baselines databases previously initialized by schema.sql.
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS event_batches (
id TEXT PRIMARY KEY NOT NULL,
received_at INTEGER NOT NULL,
install_id TEXT NOT NULL,
app_version TEXT,
platform TEXT,
event_count INTEGER NOT NULL,
payload_json TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_event_batches_received ON event_batches (received_at);
CREATE INDEX IF NOT EXISTS idx_event_batches_install ON event_batches (install_id);
CREATE TABLE IF NOT EXISTS crashes (
id TEXT PRIMARY KEY NOT NULL,
received_at INTEGER NOT NULL,
occurred_at INTEGER NOT NULL,
install_id TEXT NOT NULL,
app_version TEXT,
platform TEXT,
exception_type TEXT,
exception_message TEXT,
fingerprint TEXT NOT NULL,
diagnostics_enabled INTEGER NOT NULL DEFAULT 0,
stack_r2_key TEXT,
breadcrumbs_json TEXT,
schema_version INTEGER NOT NULL DEFAULT 1
);
CREATE INDEX IF NOT EXISTS idx_crashes_received ON crashes (received_at);
CREATE INDEX IF NOT EXISTS idx_crashes_fingerprint ON crashes (fingerprint);
CREATE INDEX IF NOT EXISTS idx_crashes_install ON crashes (install_id);
CREATE TABLE IF NOT EXISTS bugs (
id TEXT PRIMARY KEY NOT NULL,
received_at INTEGER NOT NULL,
install_id TEXT NOT NULL,
app_version TEXT,
platform TEXT,
what_happened TEXT NOT NULL,
expected TEXT NOT NULL,
steps TEXT,
contact TEXT,
logs_r2_key TEXT,
device_json TEXT,
breadcrumbs_json TEXT,
status TEXT NOT NULL DEFAULT 'open',
schema_version INTEGER NOT NULL DEFAULT 1
);
CREATE INDEX IF NOT EXISTS idx_bugs_received ON bugs (received_at);
CREATE INDEX IF NOT EXISTS idx_bugs_status ON bugs (status);

View File

@@ -0,0 +1,4 @@
ALTER TABLE bugs ADD COLUMN occurred_at INTEGER;
-- Existing reports predate this field; their receipt time is the best available value.
UPDATE bugs SET occurred_at = received_at WHERE occurred_at IS NULL;

3151
services/diagnostics-api/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,30 @@
{
"name": "vnidrop-diagnostics-api",
"private": true,
"version": "0.1.0",
"type": "module",
"engines": {
"node": ">=22.12"
},
"scripts": {
"dev": "wrangler dev",
"predeploy": "npm run check",
"deploy": "wrangler deploy",
"deploy:dry-run": "wrangler deploy --dry-run",
"db:migrate:local": "wrangler d1 migrations apply vnidrop-diagnostics --local",
"db:migrate:remote": "wrangler d1 migrations apply vnidrop-diagnostics --remote",
"test": "vitest run",
"test:watch": "vitest",
"typegen": "wrangler types",
"types:check": "wrangler types --check",
"typecheck": "tsc --noEmit && tsc --noEmit -p test/tsconfig.json",
"check": "npm run types:check && npm run typecheck && npm test && npm run deploy:dry-run"
},
"devDependencies": {
"@cloudflare/vitest-pool-workers": "^0.18.4",
"@types/node": "^26.1.1",
"typescript": "^7.0.2",
"vitest": "^4.1.10",
"wrangler": "^4.110.0"
}
}

View File

@@ -0,0 +1,267 @@
import {
normalizeBug,
normalizeCrash,
normalizeEvents,
readJsonObject,
} from "./input";
import {
type DiagnosticsEnv,
runRetention,
storeBug,
storeCrash,
storeEvents,
} from "./storage";
const DEFAULT_MAX_BODY_BYTES = 262_144;
const HARD_MAX_BODY_BYTES = 1_048_576;
const DEFAULT_MAX_EVENTS = 50;
export default {
async fetch(request: Request, env: DiagnosticsEnv, _ctx: ExecutionContext): Promise<Response> {
const requestId = crypto.randomUUID();
const url = new URL(request.url);
try {
if (request.method === "GET" && url.pathname === "/live") {
return json({ ok: true, service: "vnidrop-diagnostics", schema: 1 }, 200, requestId);
}
if (request.method === "GET" && url.pathname === "/health") {
if (await sourceRateLimited(request, url.pathname, env)) {
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
}
const authError = await authorize(request, env);
if (authError) return json({ error: authError.error }, authError.status, requestId);
return await readiness(env, requestId);
}
if (!isIngestPath(url.pathname)) {
return json({ error: "not_found" }, 404, requestId);
}
if (request.method === "OPTIONS") {
return new Response(null, {
status: 204,
headers: responseHeaders(requestId, { allow: "POST, OPTIONS" }),
});
}
if (request.method !== "POST") {
return json(
{ error: "method_not_allowed" },
405,
requestId,
{ allow: "POST, OPTIONS" },
);
}
if (await sourceRateLimited(request, url.pathname, env)) {
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
}
const authError = await authorize(request, env);
if (authError) return json({ error: authError.error }, authError.status, requestId);
if (await installRateLimited(request, url.pathname, env)) {
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
}
const maxBodyBytes = boundedPositiveInt(
env.MAX_BODY_BYTES,
DEFAULT_MAX_BODY_BYTES,
1,
HARD_MAX_BODY_BYTES,
);
const parsed = await readJsonObject(request, maxBodyBytes);
if (!parsed.ok) return json({ error: parsed.error }, parsed.status, requestId);
switch (url.pathname) {
case "/v1/events": {
const maxEvents = boundedPositiveInt(env.MAX_EVENTS_PER_BATCH, DEFAULT_MAX_EVENTS, 1, 100);
const normalized = normalizeEvents(parsed.value, maxEvents);
if (!normalized.ok) {
return json({ error: normalized.error }, normalized.status, requestId);
}
const result = await storeEvents(normalized.value, env);
return json(
{
ok: true,
id: result.id,
stored: result.stored,
duplicate: result.duplicate,
},
202,
requestId,
);
}
case "/v1/crashes": {
const normalized = normalizeCrash(parsed.value);
if (!normalized.ok) {
return json({ error: normalized.error }, normalized.status, requestId);
}
const result = await storeCrash(normalized.value, env);
return json(
{
ok: true,
id: result.id,
fingerprint: result.fingerprint,
duplicate: result.duplicate,
},
202,
requestId,
);
}
case "/v1/bugs": {
const normalized = normalizeBug(parsed.value);
if (!normalized.ok) {
return json({ error: normalized.error }, normalized.status, requestId);
}
const result = await storeBug(normalized.value, env);
return json(
{ ok: true, id: result.id, duplicate: result.duplicate },
202,
requestId,
);
}
}
} catch (error) {
console.error(
JSON.stringify({
message: "diagnostics request failed",
requestId,
method: request.method,
path: url.pathname,
error: error instanceof Error ? error.message : String(error),
}),
);
return json({ error: "internal" }, 500, requestId);
}
},
scheduled(
controller: ScheduledController,
env: DiagnosticsEnv,
ctx: ExecutionContext,
): void {
ctx.waitUntil(
runRetention(env).catch((error) => {
console.error(
JSON.stringify({
message: "diagnostics retention failed",
scheduledTime: controller.scheduledTime,
error: error instanceof Error ? error.message : String(error),
}),
);
}),
);
},
} satisfies ExportedHandler<DiagnosticsEnv>;
async function readiness(env: DiagnosticsEnv, requestId: string): Promise<Response> {
if (!env.INGEST_KEY) {
return json({ ok: false, error: "server_misconfigured" }, 503, requestId);
}
try {
await env.DB.batch([
env.DB.prepare(
"SELECT id, received_at, install_id, payload_json FROM event_batches LIMIT 1",
),
env.DB.prepare(
"SELECT id, occurred_at, stack_r2_key, breadcrumbs_json FROM crashes LIMIT 1",
),
env.DB.prepare(
"SELECT id, occurred_at, logs_r2_key, device_json FROM bugs LIMIT 1",
),
]);
return json({ ok: true, service: "vnidrop-diagnostics", schema: 1 }, 200, requestId);
} catch (error) {
console.error(
JSON.stringify({
message: "diagnostics readiness check failed",
requestId,
error: error instanceof Error ? error.message : String(error),
}),
);
return json({ ok: false, error: "dependency_unavailable" }, 503, requestId);
}
}
async function authorize(
request: Request,
env: DiagnosticsEnv,
): Promise<{ status: 401 | 503; error: "unauthorized" | "server_misconfigured" } | null> {
const expected = env.INGEST_KEY;
if (!expected) return { status: 503, error: "server_misconfigured" };
const provided = request.headers.get("x-vnidrop-key") ?? "";
if (!(await timingSafeEqual(provided, expected))) {
return { status: 401, error: "unauthorized" };
}
return null;
}
async function sourceRateLimited(
request: Request,
path: string,
env: DiagnosticsEnv,
): Promise<boolean> {
const source = request.headers.get("cf-connecting-ip")?.slice(0, 64) || "unknown";
const result = await env.SOURCE_RATE_LIMITER.limit({ key: `${source}:${path}` });
return !result.success;
}
async function installRateLimited(
request: Request,
path: string,
env: DiagnosticsEnv,
): Promise<boolean> {
const source = request.headers.get("cf-connecting-ip")?.slice(0, 64) || "unknown";
const installId = request.headers.get("x-vnidrop-install-id")?.trim().slice(0, 80) || source;
const result = await env.INSTALL_RATE_LIMITER.limit({ key: `${installId}:${path}` });
return !result.success;
}
function isIngestPath(path: string): path is "/v1/events" | "/v1/crashes" | "/v1/bugs" {
return path === "/v1/events" || path === "/v1/crashes" || path === "/v1/bugs";
}
async function timingSafeEqual(provided: string, expected: string): Promise<boolean> {
const encoder = new TextEncoder();
const [providedHash, expectedHash] = await Promise.all([
crypto.subtle.digest("SHA-256", encoder.encode(provided)),
crypto.subtle.digest("SHA-256", encoder.encode(expected)),
]);
return crypto.subtle.timingSafeEqual(providedHash, expectedHash);
}
function json(
body: unknown,
status: number,
requestId: string,
extraHeaders?: Readonly<Record<string, string>>,
): Response {
return new Response(JSON.stringify(body), {
status,
headers: responseHeaders(requestId, {
"content-type": "application/json; charset=utf-8",
...extraHeaders,
}),
});
}
function responseHeaders(
requestId: string,
extraHeaders?: Readonly<Record<string, string>>,
): Headers {
const headers = new Headers(extraHeaders);
headers.set("cache-control", "no-store");
headers.set("x-content-type-options", "nosniff");
headers.set("x-request-id", requestId);
return headers;
}
function boundedPositiveInt(
raw: string | undefined,
fallback: number,
minimum: number,
maximum: number,
): number {
const parsed = Number(raw);
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) return fallback;
return parsed;
}

View File

@@ -0,0 +1,593 @@
export type JsonObject = Record<string, unknown>;
export type InputFailure = {
ok: false;
status: 400 | 413 | 415;
error: string;
};
export type InputResult<T> = { ok: true; value: T } | InputFailure;
export type NormalizedProperties = Record<string, string>;
export interface NormalizedEvent {
name: string;
timestampMillis: number;
properties: NormalizedProperties;
schemaVersion: 1;
}
export interface NormalizedBreadcrumb {
name: string;
timestampMillis: number;
properties: NormalizedProperties;
}
export interface NormalizedDevice {
deviceName: string;
deviceModel: string;
operatingSystem: string;
network: string;
batteryLevel: string;
}
export interface NormalizedEventsPayload {
batchId: string;
installId: string;
appVersion: string;
platform: string;
events: NormalizedEvent[];
}
export interface NormalizedCrashPayload {
id: string;
installId: string;
appVersion: string;
platform: string;
exceptionType: string;
exceptionMessage: string;
stackTrace: string;
occurredAt: number;
diagnosticsEnabledAtCapture: boolean;
breadcrumbs: NormalizedBreadcrumb[];
schemaVersion: 1;
}
export interface NormalizedBugPayload {
id: string;
installId: string;
appVersion: string;
platform: string;
occurredAt: number;
whatHappened: string;
expected: string;
steps: string;
contact: string;
logs: string;
device: NormalizedDevice;
breadcrumbs: NormalizedBreadcrumb[];
schemaVersion: 1;
}
export const MAX_LOG_BYTES = 192 * 1024;
export const MAX_BREADCRUMBS_JSON_BYTES = 16_000;
export const MAX_DEVICE_JSON_BYTES = 4_000;
const MAX_PROPERTIES = 12;
const MAX_BREADCRUMBS = 40;
const MISSING = Symbol("missing");
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
const UTF8_ENCODER = new TextEncoder();
export async function readJsonObject(
request: Request,
maxBytes: number,
): Promise<InputResult<JsonObject>> {
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
throw new RangeError("maxBytes must be a positive safe integer");
}
if (!isApplicationJson(request.headers.get("content-type"))) {
return failure(415, "unsupported_media_type");
}
const contentLength = request.headers.get("content-length");
if (contentLength != null) {
const trimmed = contentLength.trim();
if (!/^\d+$/.test(trimmed)) {
return failure(400, "invalid_content_length");
}
const normalizedLength = trimmed.replace(/^0+(?=\d)/, "");
const maxLength = String(maxBytes);
if (
normalizedLength.length > maxLength.length ||
(normalizedLength.length === maxLength.length && normalizedLength > maxLength)
) {
return failure(413, "payload_too_large");
}
const declaredBytes = Number(trimmed);
if (!Number.isSafeInteger(declaredBytes)) {
return failure(400, "invalid_content_length");
}
}
if (request.body == null) {
return failure(400, "invalid_json");
}
let reader: ReadableStreamDefaultReader<Uint8Array>;
try {
reader = request.body.getReader();
} catch {
return failure(400, "invalid_body");
}
const chunks: Uint8Array[] = [];
let totalBytes = 0;
try {
while (true) {
const chunk = await reader.read();
if (chunk.done) break;
if (chunk.value.byteLength > maxBytes - totalBytes) {
try {
await reader.cancel("payload_too_large");
} catch {
// The size failure remains authoritative if cancellation also fails.
}
return failure(413, "payload_too_large");
}
totalBytes += chunk.value.byteLength;
chunks.push(chunk.value);
}
} catch {
return failure(400, "invalid_body");
} finally {
reader.releaseLock();
}
const bytes = joinChunks(chunks, totalBytes);
let raw: string;
try {
raw = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
} catch {
return failure(400, "invalid_utf8");
}
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
return failure(400, "invalid_json");
}
if (!isPlainObject(parsed)) {
return failure(400, "invalid_body");
}
return success(parsed);
}
export function normalizeEvents(
body: JsonObject,
maxEvents = 50,
): InputResult<NormalizedEventsPayload> {
if (!isPlainObject(body)) return failure(400, "invalid_body");
if (!Number.isSafeInteger(maxEvents) || maxEvents <= 0) {
throw new RangeError("maxEvents must be a positive safe integer");
}
const batchId = idField(body, ["batchId", "batch_id"], "invalid_batch_id");
if (!batchId.ok) return batchId;
const installId = installIdField(body);
if (!installId.ok) return installId;
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
if (!appVersion.ok) return appVersion;
const platform = stringField(body, ["platform"], 40, "invalid_platform");
if (!platform.ok) return platform;
const batchSchema = schemaVersion(body);
if (!batchSchema.ok) return batchSchema;
const rawEvents = pick(body, ["events"]);
if (!Array.isArray(rawEvents)) return failure(400, "invalid_events");
if (rawEvents.length === 0) return failure(400, "empty_batch");
if (rawEvents.length > maxEvents) return failure(400, "batch_too_large");
const events: NormalizedEvent[] = [];
for (const rawEvent of rawEvents) {
const event = normalizeEvent(rawEvent);
if (!event.ok) return event;
events.push(event.value);
}
return success({
batchId: batchId.value,
installId: installId.value,
appVersion: appVersion.value,
platform: platform.value,
events,
});
}
export function normalizeCrash(body: JsonObject): InputResult<NormalizedCrashPayload> {
if (!isPlainObject(body)) return failure(400, "invalid_body");
const id = idField(body, ["id"], "invalid_id");
if (!id.ok) return id;
const installId = installIdField(body);
if (!installId.ok) return installId;
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
if (!appVersion.ok) return appVersion;
const platform = stringField(body, ["platform"], 40, "invalid_platform");
if (!platform.ok) return platform;
const exceptionType = stringField(
body,
["exceptionType", "exception_type"],
120,
"invalid_exception_type",
true,
true,
);
if (!exceptionType.ok) return exceptionType;
const exceptionMessage = stringField(
body,
["exceptionMessage", "exception_message"],
2_000,
"invalid_exception_message",
true,
);
if (!exceptionMessage.ok) return exceptionMessage;
const stackTrace = stringField(
body,
["stackTrace", "stack_trace"],
32_000,
"invalid_stack_trace",
true,
);
if (!stackTrace.ok) return stackTrace;
const occurredAt = timestampField(
body,
["timestampMillis", "timestamp_millis", "occurredAt", "occurred_at"],
);
if (!occurredAt.ok) return occurredAt;
const diagnosticsEnabled = booleanField(
body,
[
"diagnosticsEnabledAtCapture",
"diagnostics_enabled_at_capture",
"diagnostics_enabled",
],
"invalid_diagnostics_enabled",
true,
);
if (!diagnosticsEnabled.ok) return diagnosticsEnabled;
const version = schemaVersion(body);
if (!version.ok) return version;
const breadcrumbs = normalizeBreadcrumbs(pick(body, ["breadcrumbs"]));
if (!breadcrumbs.ok) return breadcrumbs;
return success({
id: id.value,
installId: installId.value,
appVersion: appVersion.value,
platform: platform.value,
exceptionType: exceptionType.value,
exceptionMessage: exceptionMessage.value,
stackTrace: stackTrace.value,
occurredAt: occurredAt.value,
diagnosticsEnabledAtCapture: diagnosticsEnabled.value,
breadcrumbs: breadcrumbs.value,
schemaVersion: version.value,
});
}
export function normalizeBug(body: JsonObject): InputResult<NormalizedBugPayload> {
if (!isPlainObject(body)) return failure(400, "invalid_body");
const id = idField(body, ["id"], "invalid_id");
if (!id.ok) return id;
const installId = installIdField(body);
if (!installId.ok) return installId;
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
if (!appVersion.ok) return appVersion;
const platform = stringField(body, ["platform"], 40, "invalid_platform");
if (!platform.ok) return platform;
const occurredAt = timestampField(
body,
["timestampMillis", "timestamp_millis", "occurredAt", "occurred_at"],
);
if (!occurredAt.ok) return occurredAt;
const whatHappened = stringField(
body,
["whatHappened", "what_happened"],
4_000,
"missing_fields",
true,
true,
);
if (!whatHappened.ok) return whatHappened;
const expected = stringField(body, ["expected"], 4_000, "missing_fields", true, true);
if (!expected.ok) return expected;
const steps = stringField(body, ["steps"], 4_000, "invalid_steps");
if (!steps.ok) return steps;
const contact = stringField(body, ["contact"], 320, "invalid_contact");
if (!contact.ok) return contact;
const includeLogs = booleanField(
body,
["includeLogs", "include_logs"],
"invalid_include_logs",
false,
);
if (!includeLogs.ok) return includeLogs;
const logs = stringField(body, ["logs"], MAX_LOG_BYTES, "invalid_logs");
if (!logs.ok) return logs;
const device = normalizeDevice(pick(body, ["device"]));
if (!device.ok) return device;
const breadcrumbs = normalizeBreadcrumbs(pick(body, ["breadcrumbs"]));
if (!breadcrumbs.ok) return breadcrumbs;
const version = schemaVersion(body);
if (!version.ok) return version;
return success({
id: id.value,
installId: installId.value,
appVersion: appVersion.value,
platform: platform.value,
occurredAt: occurredAt.value,
whatHappened: whatHappened.value,
expected: expected.value,
steps: steps.value,
contact: contact.value,
logs: includeLogs.value === true ? logs.value : "",
device: device.value,
breadcrumbs: breadcrumbs.value,
schemaVersion: version.value,
});
}
function normalizeEvent(raw: unknown): InputResult<NormalizedEvent> {
if (!isPlainObject(raw)) return failure(400, "invalid_event");
const name = stringField(raw, ["name"], 64, "invalid_event", true, true);
if (!name.ok) return name;
const timestamp = timestampField(raw, ["timestampMillis", "timestamp_millis", "ts"]);
if (!timestamp.ok) return failure(400, "invalid_event");
const properties = normalizeProperties(pick(raw, ["properties", "props"]), "invalid_event");
if (!properties.ok) return properties;
const version = schemaVersion(raw);
if (!version.ok) return version;
return success({
name: name.value,
timestampMillis: timestamp.value,
properties: properties.value,
schemaVersion: version.value,
});
}
function normalizeBreadcrumbs(raw: unknown | typeof MISSING): InputResult<NormalizedBreadcrumb[]> {
if (raw === MISSING) return success([]);
if (!Array.isArray(raw)) return failure(400, "invalid_breadcrumbs");
const breadcrumbs: NormalizedBreadcrumb[] = [];
for (const item of raw.slice(0, MAX_BREADCRUMBS)) {
if (!isPlainObject(item)) return failure(400, "invalid_breadcrumbs");
const name = stringField(item, ["name"], 64, "invalid_breadcrumbs", true, true);
if (!name.ok) return name;
const timestamp = timestampField(item, ["timestampMillis", "timestamp_millis", "ts"]);
if (!timestamp.ok) return failure(400, "invalid_breadcrumbs");
const properties = normalizeProperties(
pick(item, ["properties", "props"]),
"invalid_breadcrumbs",
);
if (!properties.ok) return properties;
breadcrumbs.push({
name: name.value,
timestampMillis: timestamp.value,
properties: properties.value,
});
if (jsonBytes(breadcrumbs) > MAX_BREADCRUMBS_JSON_BYTES) {
breadcrumbs.pop();
break;
}
}
return success(breadcrumbs);
}
function normalizeProperties(
raw: unknown | typeof MISSING,
error: string,
): InputResult<NormalizedProperties> {
if (raw === MISSING) return success({});
if (!isPlainObject(raw)) return failure(400, error);
const entries: Array<[string, string]> = [];
const normalizedKeys = new Set<string>();
for (const [key, value] of Object.entries(raw).slice(0, MAX_PROPERTIES)) {
if (typeof value !== "string") return failure(400, error);
const normalizedKey = truncateUtf8(key, 40);
if (normalizedKey.length === 0 || normalizedKeys.has(normalizedKey)) {
return failure(400, error);
}
normalizedKeys.add(normalizedKey);
entries.push([normalizedKey, truncateUtf8(value, 128)]);
}
return success(Object.fromEntries(entries));
}
function normalizeDevice(raw: unknown | typeof MISSING): InputResult<NormalizedDevice> {
if (raw === MISSING) raw = {};
if (!isPlainObject(raw)) return failure(400, "invalid_device");
const deviceName = stringField(raw, ["deviceName", "device_name"], 128, "invalid_device");
if (!deviceName.ok) return deviceName;
const deviceModel = stringField(raw, ["deviceModel", "device_model"], 128, "invalid_device");
if (!deviceModel.ok) return deviceModel;
const operatingSystem = stringField(
raw,
["operatingSystem", "operating_system"],
192,
"invalid_device",
);
if (!operatingSystem.ok) return operatingSystem;
const network = stringField(raw, ["network"], 96, "invalid_device");
if (!network.ok) return network;
const batteryLevel = stringField(raw, ["batteryLevel", "battery_level"], 64, "invalid_device");
if (!batteryLevel.ok) return batteryLevel;
const device: NormalizedDevice = {
deviceName: deviceName.value,
deviceModel: deviceModel.value,
operatingSystem: operatingSystem.value,
network: network.value,
batteryLevel: batteryLevel.value,
};
if (jsonBytes(device) > MAX_DEVICE_JSON_BYTES) {
return failure(400, "invalid_device");
}
return success(device);
}
function stringField(
object: JsonObject,
keys: readonly string[],
maxBytes: number,
error: string,
required = false,
nonEmpty = false,
): InputResult<string> {
const raw = pick(object, keys);
if (raw === MISSING) {
return required ? failure(400, error) : success("");
}
if (typeof raw !== "string") return failure(400, error);
const value = truncateUtf8(raw, maxBytes);
if (nonEmpty && value.trim().length === 0) return failure(400, error);
return success(value);
}
function idField(
object: JsonObject,
keys: readonly string[],
error: string,
): InputResult<string> {
const raw = pick(object, keys);
if (typeof raw !== "string" || !UUID_PATTERN.test(raw)) return failure(400, error);
return success(raw.toLowerCase());
}
function installIdField(object: JsonObject): InputResult<string> {
const raw = pick(object, ["installId", "install_id"]);
if (raw === MISSING || raw === "") return success("unknown");
if (
typeof raw !== "string" ||
raw.trim().length === 0 ||
/[\u0000-\u001f\u007f]/.test(raw)
) {
return failure(400, "invalid_install_id");
}
return success(truncateUtf8(raw, 80));
}
function timestampField(object: JsonObject, keys: readonly string[]): InputResult<number> {
const raw = pick(object, keys);
if (typeof raw !== "number" || !Number.isSafeInteger(raw) || raw < 0) {
return failure(400, "invalid_timestamp");
}
return success(raw);
}
function booleanField(
object: JsonObject,
keys: readonly string[],
error: string,
required: true,
): InputResult<boolean>;
function booleanField(
object: JsonObject,
keys: readonly string[],
error: string,
required: false,
): InputResult<boolean | undefined>;
function booleanField(
object: JsonObject,
keys: readonly string[],
error: string,
required: boolean,
): InputResult<boolean | undefined> {
const raw = pick(object, keys);
if (raw === MISSING) {
return required ? failure(400, error) : success(undefined);
}
return typeof raw === "boolean" ? success(raw) : failure(400, error);
}
function schemaVersion(object: JsonObject): InputResult<1> {
const raw = pick(object, ["schemaVersion", "schema_version"]);
if (raw === MISSING || raw === 1) return success(1);
return failure(400, "unsupported_schema_version");
}
function pick(object: JsonObject, keys: readonly string[]): unknown | typeof MISSING {
for (const key of keys) {
if (Object.prototype.hasOwnProperty.call(object, key)) return object[key];
}
return MISSING;
}
function isPlainObject(value: unknown): value is JsonObject {
if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
const prototype = Object.getPrototypeOf(value);
return prototype === Object.prototype || prototype === null;
}
function isApplicationJson(contentType: string | null): boolean {
if (contentType == null) return false;
const [mediaType, ...parameters] = contentType.split(";");
if (mediaType?.trim().toLowerCase() !== "application/json") return false;
for (const parameter of parameters) {
const separator = parameter.indexOf("=");
if (separator < 0) continue;
if (parameter.slice(0, separator).trim().toLowerCase() !== "charset") continue;
const charset = parameter
.slice(separator + 1)
.trim()
.replace(/^"(.*)"$/, "$1")
.toLowerCase();
if (charset !== "utf-8" && charset !== "utf8") return false;
}
return true;
}
function joinChunks(chunks: readonly Uint8Array[], totalBytes: number): Uint8Array {
if (chunks.length === 1) return chunks[0] ?? new Uint8Array();
const bytes = new Uint8Array(totalBytes);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
return bytes;
}
function truncateUtf8(value: string, maxBytes: number): string {
const bytes = UTF8_ENCODER.encode(value);
if (bytes.byteLength <= maxBytes) return value;
for (let end = maxBytes; end >= Math.max(0, maxBytes - 3); end -= 1) {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.subarray(0, end),
);
} catch {
// A UTF-8 boundary is at most three bytes behind the byte cap.
}
}
return "";
}
function jsonBytes(value: unknown): number {
return UTF8_ENCODER.encode(JSON.stringify(value)).byteLength;
}
function success<T>(value: T): InputResult<T> {
return { ok: true, value };
}
function failure(status: 400 | 413 | 415, error: string): InputFailure {
return { ok: false, status, error };
}

View File

@@ -0,0 +1,340 @@
import type {
NormalizedBugPayload,
NormalizedCrashPayload,
NormalizedEventsPayload,
} from "./input";
export type DiagnosticsEnv = Cloudflare.Env & {
INGEST_KEY?: string;
AE?: AnalyticsEngineDataset;
};
export interface StoreResult {
id: string;
duplicate: boolean;
stored: number;
}
export async function storeEvents(
payload: NormalizedEventsPayload,
env: DiagnosticsEnv,
): Promise<StoreResult> {
const result = await env.DB.prepare(
`INSERT INTO event_batches (id, received_at, install_id, app_version, platform, event_count, payload_json)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO NOTHING`,
)
.bind(
payload.batchId,
Date.now(),
payload.installId,
payload.appVersion,
payload.platform,
payload.events.length,
JSON.stringify(payload.events),
)
.run();
const duplicate = result.meta.changes === 0;
if (!duplicate && env.AE) {
try {
for (const event of payload.events) {
env.AE.writeDataPoint({
blobs: [
event.name,
payload.platform,
payload.appVersion,
payload.installId,
JSON.stringify(event.properties),
payload.batchId,
],
doubles: [event.timestampMillis, event.schemaVersion],
indexes: [payload.installId],
});
}
} catch (error) {
// D1 remains the durable source of truth if the optional analytics index is unavailable.
console.error(
JSON.stringify({
message: "failed to index diagnostics event batch",
batchId: payload.batchId,
error: error instanceof Error ? error.message : String(error),
}),
);
}
}
return {
id: payload.batchId,
duplicate,
stored: duplicate ? 0 : payload.events.length,
};
}
export async function storeCrash(
payload: NormalizedCrashPayload,
env: DiagnosticsEnv,
): Promise<StoreResult & { fingerprint: string }> {
const database = env.DB.withSession("first-primary");
const existing = await database
.prepare("SELECT fingerprint FROM crashes WHERE id = ?")
.bind(payload.id)
.first<{ fingerprint: string }>();
if (existing) {
return { id: payload.id, duplicate: true, stored: 0, fingerprint: existing.fingerprint };
}
const fingerprint = await crashFingerprint(payload.exceptionType, payload.stackTrace);
const stackKey = payload.stackTrace
? `crashes/${payload.id}/${crypto.randomUUID()}/stack.txt`
: null;
if (stackKey) {
await env.BLOBS.put(stackKey, payload.stackTrace, {
httpMetadata: { contentType: "text/plain; charset=utf-8" },
customMetadata: { installId: payload.installId, fingerprint },
});
}
try {
const result = await database
.prepare(
`INSERT INTO crashes (
id, received_at, occurred_at, install_id, app_version, platform,
exception_type, exception_message, fingerprint, diagnostics_enabled,
stack_r2_key, breadcrumbs_json, schema_version
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO NOTHING`,
)
.bind(
payload.id,
Date.now(),
payload.occurredAt,
payload.installId,
payload.appVersion,
payload.platform,
payload.exceptionType,
payload.exceptionMessage,
fingerprint,
payload.diagnosticsEnabledAtCapture ? 1 : 0,
stackKey,
JSON.stringify(payload.breadcrumbs),
payload.schemaVersion,
)
.run();
const duplicate = result.meta.changes === 0;
if (duplicate) {
const stored = await database
.prepare("SELECT fingerprint FROM crashes WHERE id = ?")
.bind(payload.id)
.first<{ fingerprint: string }>();
if (!stored) throw new Error("duplicate crash row was not readable");
if (stackKey) await deleteAttemptBlob(env, stackKey);
return { id: payload.id, duplicate: true, stored: 0, fingerprint: stored.fingerprint };
}
return { id: payload.id, duplicate: false, stored: 1, fingerprint };
} catch (error) {
if (stackKey) {
await deleteAttemptBlob(env, stackKey);
}
throw error;
}
}
export async function storeBug(
payload: NormalizedBugPayload,
env: DiagnosticsEnv,
): Promise<StoreResult> {
const database = env.DB.withSession("first-primary");
const existing = await database
.prepare("SELECT id FROM bugs WHERE id = ?")
.bind(payload.id)
.first<{ id: string }>();
if (existing) return { id: payload.id, duplicate: true, stored: 0 };
const logsKey = payload.logs ? `bugs/${payload.id}/${crypto.randomUUID()}/logs.txt` : null;
if (logsKey) {
await env.BLOBS.put(logsKey, payload.logs, {
httpMetadata: { contentType: "text/plain; charset=utf-8" },
customMetadata: { installId: payload.installId },
});
}
try {
const result = await database
.prepare(
`INSERT INTO bugs (
id, received_at, occurred_at, install_id, app_version, platform,
what_happened, expected, steps, contact, logs_r2_key,
device_json, breadcrumbs_json, status, schema_version
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?)
ON CONFLICT(id) DO NOTHING`,
)
.bind(
payload.id,
Date.now(),
payload.occurredAt,
payload.installId,
payload.appVersion,
payload.platform,
payload.whatHappened,
payload.expected,
payload.steps,
payload.contact,
logsKey,
JSON.stringify(payload.device),
JSON.stringify(payload.breadcrumbs),
payload.schemaVersion,
)
.run();
const duplicate = result.meta.changes === 0;
if (duplicate && logsKey) {
await deleteAttemptBlob(env, logsKey);
}
return { id: payload.id, duplicate, stored: duplicate ? 0 : 1 };
} catch (error) {
if (logsKey) {
await deleteAttemptBlob(env, logsKey);
}
throw error;
}
}
export async function runRetention(env: DiagnosticsEnv): Promise<void> {
const retentionDays = boundedPositiveInt(env.RETENTION_DAYS, 90, 1, 3_650);
const cutoff = Date.now() - retentionDays * 86_400_000;
// Eight full passes plus the backlog check use at most 43 of D1's 50 queries per invocation.
for (let pass = 0; pass < 8; pass += 1) {
const hasFullBatch = await runRetentionPass(env, cutoff);
if (!hasFullBatch) return;
}
const [events, crashes, bugs] = await env.DB.batch<{ count: number }>([
env.DB.prepare("SELECT COUNT(*) AS count FROM event_batches WHERE received_at < ?").bind(
cutoff,
),
env.DB.prepare("SELECT COUNT(*) AS count FROM crashes WHERE received_at < ?").bind(cutoff),
env.DB.prepare("SELECT COUNT(*) AS count FROM bugs WHERE received_at < ?").bind(cutoff),
]);
console.warn(
JSON.stringify({
message: "diagnostics retention reached its per-run pass limit",
cutoff,
backlog: {
eventBatches: events.results[0]?.count ?? 0,
crashes: crashes.results[0]?.count ?? 0,
bugs: bugs.results[0]?.count ?? 0,
},
}),
);
}
async function runRetentionPass(env: DiagnosticsEnv, cutoff: number): Promise<boolean> {
const reportBatchSize = 900;
const eventBatchSize = 1_000;
const [crashes, bugs] = await Promise.all([
expiredBlobRows(env.DB, "crashes", "stack_r2_key", cutoff, reportBatchSize),
expiredBlobRows(env.DB, "bugs", "logs_r2_key", cutoff, reportBatchSize),
]);
const blobKeys = [...crashes, ...bugs]
.map((row) => row.blobKey)
.filter((key): key is string => key !== null);
for (let offset = 0; offset < blobKeys.length; offset += 1_000) {
await env.BLOBS.delete(blobKeys.slice(offset, offset + 1_000));
}
const statements = [retentionStatement(env.DB, "event_batches", cutoff, eventBatchSize)];
if (crashes.length > 0) statements.push(deleteRowsById(env.DB, "crashes", crashes));
if (bugs.length > 0) statements.push(deleteRowsById(env.DB, "bugs", bugs));
const [eventsResult] = await env.DB.batch(statements);
return (
eventsResult.meta.changes === eventBatchSize ||
crashes.length === reportBatchSize ||
bugs.length === reportBatchSize
);
}
interface ExpiredBlobRow {
id: string;
blobKey: string | null;
}
async function expiredBlobRows(
database: D1Database,
table: "crashes" | "bugs",
column: "stack_r2_key" | "logs_r2_key",
cutoff: number,
batchSize: number,
): Promise<ExpiredBlobRow[]> {
const result = await database
.prepare(
`SELECT id, ${column} AS blobKey
FROM ${table}
WHERE received_at < ?
ORDER BY received_at
LIMIT ?`,
)
.bind(cutoff, batchSize)
.all<ExpiredBlobRow>();
return result.results;
}
function retentionStatement(
database: D1Database,
table: "event_batches" | "crashes" | "bugs",
cutoff: number,
batchSize: number,
): D1PreparedStatement {
return database
.prepare(
`DELETE FROM ${table}
WHERE rowid IN (
SELECT rowid FROM ${table} WHERE received_at < ? ORDER BY received_at LIMIT ?
)`,
)
.bind(cutoff, batchSize);
}
function deleteRowsById(
database: D1Database,
table: "crashes" | "bugs",
rows: ExpiredBlobRow[],
): D1PreparedStatement {
return database
.prepare(`DELETE FROM ${table} WHERE id IN (SELECT value FROM json_each(?))`)
.bind(JSON.stringify(rows.map((row) => row.id)));
}
async function crashFingerprint(exceptionType: string, stackTrace: string): Promise<string> {
const topFrames = stackTrace
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.slice(0, 4)
.join("\n");
const bytes = new TextEncoder().encode(`${exceptionType}\n${topFrames}`);
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes));
return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join("");
}
async function deleteAttemptBlob(env: DiagnosticsEnv, key: string): Promise<void> {
try {
await env.BLOBS.delete(key);
} catch (error) {
console.error(
JSON.stringify({
message: "failed to remove uncommitted diagnostics blob",
error: error instanceof Error ? error.message : String(error),
}),
);
}
}
function boundedPositiveInt(
raw: string | undefined,
fallback: number,
minimum: number,
maximum: number,
): number {
const parsed = Number(raw);
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) return fallback;
return parsed;
}

View File

@@ -0,0 +1,4 @@
import { env } from "cloudflare:workers";
import { applyD1Migrations } from "cloudflare:test";
await applyD1Migrations(env.DB, env.TEST_MIGRATIONS);

12
services/diagnostics-api/test/env.d.ts vendored Normal file
View File

@@ -0,0 +1,12 @@
import type { D1Migration } from "@cloudflare/vitest-pool-workers";
declare global {
namespace Cloudflare {
interface Env {
INGEST_KEY: string;
TEST_MIGRATIONS: D1Migration[];
}
}
}
export {};

View File

@@ -0,0 +1,266 @@
import { describe, expect, it } from "vitest";
import {
MAX_BREADCRUMBS_JSON_BYTES,
MAX_DEVICE_JSON_BYTES,
MAX_LOG_BYTES,
normalizeBug,
normalizeCrash,
normalizeEvents,
readJsonObject,
} from "../src/input";
const ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
const INSTALL_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
const ENCODER = new TextEncoder();
describe("readJsonObject", () => {
it("enforces the byte limit on a chunked body without Content-Length", async () => {
const bytes = ENCODER.encode(JSON.stringify({ value: "😀".repeat(40) }));
const request = chunkedJsonRequest([
bytes.subarray(0, 7),
bytes.subarray(7, 31),
bytes.subarray(31),
]);
expect(request.headers.has("content-length")).toBe(false);
await expect(readJsonObject(request, 64)).resolves.toEqual({
ok: false,
status: 413,
error: "payload_too_large",
});
});
it("joins chunks before fatally decoding UTF-8", async () => {
const bytes = ENCODER.encode(JSON.stringify({ value: "😀" }));
const emojiStart = bytes.indexOf(0xf0);
const request = chunkedJsonRequest([
bytes.subarray(0, emojiStart + 2),
bytes.subarray(emojiStart + 2),
]);
await expect(readJsonObject(request, bytes.byteLength)).resolves.toEqual({
ok: true,
value: { value: "😀" },
});
});
it("rejects malformed UTF-8", async () => {
const request = chunkedJsonRequest([
new Uint8Array([0x7b, 0x22, 0x78, 0x22, 0x3a, 0x22, 0xc3, 0x28, 0x22, 0x7d]),
]);
await expect(readJsonObject(request, 100)).resolves.toEqual({
ok: false,
status: 400,
error: "invalid_utf8",
});
});
it("requires application/json with a UTF-8 charset", async () => {
const missing = new Request("https://example.test/v1/events", {
method: "POST",
body: "{}",
});
const wrongCharset = chunkedJsonRequest([ENCODER.encode("{}")], "application/json; charset=utf-16");
await expect(readJsonObject(missing, 100)).resolves.toEqual({
ok: false,
status: 415,
error: "unsupported_media_type",
});
await expect(readJsonObject(wrongCharset, 100)).resolves.toEqual({
ok: false,
status: 415,
error: "unsupported_media_type",
});
});
it("validates Content-Length before reading the body", async () => {
const invalid = chunkedJsonRequest([ENCODER.encode("{}")], "application/json", "invalid");
const oversized = chunkedJsonRequest(
[ENCODER.encode("{}")],
"application/json",
"999999999999999999999999999999999999",
);
await expect(readJsonObject(invalid, 100)).resolves.toEqual({
ok: false,
status: 400,
error: "invalid_content_length",
});
await expect(readJsonObject(oversized, 100)).resolves.toEqual({
ok: false,
status: 413,
error: "payload_too_large",
});
});
it("rejects a non-object JSON root", async () => {
const request = chunkedJsonRequest([ENCODER.encode("null")]);
await expect(readJsonObject(request, 100)).resolves.toEqual({
ok: false,
status: 400,
error: "invalid_body",
});
});
});
describe("normalizers", () => {
it("preserves false booleans and rejects their string representation", () => {
const crash = crashPayload(false);
const normalizedCrash = normalizeCrash(crash);
expect(normalizedCrash.ok).toBe(true);
if (normalizedCrash.ok) {
expect(normalizedCrash.value.diagnosticsEnabledAtCapture).toBe(false);
}
expect(normalizeCrash(crashPayload("false"))).toEqual({
ok: false,
status: 400,
error: "invalid_diagnostics_enabled",
});
const bug = bugPayload({ include_logs: false, logs: "discard me" });
const normalizedBug = normalizeBug(bug);
expect(normalizedBug.ok).toBe(true);
if (normalizedBug.ok) expect(normalizedBug.value.logs).toBe("");
const missingConsent = normalizeBug(bugPayload({ logs: "discard me" }));
expect(missingConsent.ok && missingConsent.value.logs).toBe("");
expect(normalizeBug(bugPayload({ include_logs: "false" }))).toEqual({
ok: false,
status: 400,
error: "invalid_include_logs",
});
});
it("keeps logs, breadcrumbs, and device JSON within valid byte budgets", () => {
const properties = Object.fromEntries(
Array.from({ length: 12 }, (_, index) => [`key-${index}-${"\u0000".repeat(40)}`, "\u0000".repeat(128)]),
);
const breadcrumbs = Array.from({ length: 40 }, (_, index) => ({
name: `crumb-${index}`,
timestamp_millis: index,
properties,
}));
const result = normalizeBug(
bugPayload({
include_logs: true,
logs: "😀".repeat(60_000),
breadcrumbs,
device: {
device_name: "\u0000".repeat(200),
device_model: "\u0000".repeat(200),
operating_system: "\u0000".repeat(300),
network: "\u0000".repeat(150),
battery_level: "\u0000".repeat(100),
},
}),
);
expect(result.ok).toBe(true);
if (!result.ok) return;
const breadcrumbsJson = JSON.stringify(result.value.breadcrumbs);
const deviceJson = JSON.stringify(result.value.device);
expect(ENCODER.encode(result.value.logs).byteLength).toBe(MAX_LOG_BYTES);
expect(ENCODER.encode(breadcrumbsJson).byteLength).toBeLessThanOrEqual(
MAX_BREADCRUMBS_JSON_BYTES,
);
expect(ENCODER.encode(deviceJson).byteLength).toBeLessThanOrEqual(MAX_DEVICE_JSON_BYTES);
expect(JSON.parse(breadcrumbsJson)).toEqual(result.value.breadcrumbs);
expect(JSON.parse(deviceJson)).toEqual(result.value.device);
});
it("requires stable report IDs and validates supplied IDs and schema versions", () => {
const result = normalizeEvents({
events: [{ name: "opened", ts: 1, schema_version: 1 }],
});
expect(result).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
const legacyInstall = normalizeEvents({
batch_id: ID,
install_id: "legacy-test-install",
events: [{ name: "opened", ts: 1 }],
});
expect(legacyInstall.ok && legacyInstall.value.installId).toBe("legacy-test-install");
const missingInstall = normalizeEvents({
batch_id: ID,
events: [{ name: "opened", ts: 1 }],
});
expect(missingInstall.ok && missingInstall.value.installId).toBe("unknown");
expect(
normalizeEvents({
batch_id: ID,
install_id: "bad\u0000install",
events: [{ name: "opened", ts: 1 }],
}),
).toEqual({ ok: false, status: 400, error: "invalid_install_id" });
expect(
normalizeEvents({
batch_id: "not-a-uuid",
events: [{ name: "opened", timestamp_millis: 1 }],
}),
).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
expect(
normalizeEvents({
batch_id: ID,
install_id: INSTALL_ID,
events: [{ name: "opened", timestamp_millis: 1, schema_version: 2 }],
}),
).toEqual({ ok: false, status: 400, error: "unsupported_schema_version" });
});
});
function chunkedJsonRequest(
chunks: readonly Uint8Array[],
contentType = "application/json; charset=utf-8",
contentLength?: string,
): Request {
return new Request("https://example.test/v1/events", {
method: "POST",
headers: {
"content-type": contentType,
...(contentLength == null ? {} : { "content-length": contentLength }),
},
body: new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) controller.enqueue(chunk);
controller.close();
},
}),
});
}
function crashPayload(diagnosticsEnabled: unknown): Record<string, unknown> {
return {
id: ID,
install_id: INSTALL_ID,
app_version: "1.0",
platform: "test",
exception_type: "ExampleError",
exception_message: "message",
stack_trace: "stack",
occurred_at: 1,
diagnostics_enabled: diagnosticsEnabled,
schema_version: 1,
breadcrumbs: [],
};
}
function bugPayload(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
id: ID,
install_id: INSTALL_ID,
app_version: "1.0",
platform: "test",
occurred_at: 1,
what_happened: "It failed",
expected: "It worked",
steps: "Open the app",
contact: "",
logs: "",
device: {},
breadcrumbs: [],
schema_version: 1,
...overrides,
};
}

View File

@@ -0,0 +1,7 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"types": ["@cloudflare/vitest-pool-workers/types", "node"]
},
"include": ["../worker-configuration.d.ts", "../vitest.config.ts", "./**/*.ts"]
}

View File

@@ -0,0 +1,606 @@
import { env, exports } from "cloudflare:workers";
import { createExecutionContext } from "cloudflare:test";
import { describe, expect, it, vi } from "vitest";
import worker from "../src/index";
import type {
NormalizedBugPayload,
NormalizedCrashPayload,
NormalizedEventsPayload,
} from "../src/input";
import {
type DiagnosticsEnv,
runRetention,
storeBug,
storeCrash,
storeEvents,
} from "../src/storage";
const INSTALL_ID = "10000000-0000-4000-8000-000000000000";
describe("diagnostics Worker", () => {
it("keeps public routing narrow and fails closed", async () => {
const live = await exports.default.fetch(new Request("https://diagnostics.test/live"));
expect(live.status).toBe(200);
expect(await live.json()).toMatchObject({ ok: true, schema: 1 });
const health = await exports.default.fetch(healthRequest());
expect(health.status).toBe(200);
expect(await health.json()).toMatchObject({ ok: true });
const unauthorizedHealth = await exports.default.fetch(healthRequest("wrong-key"));
expect(unauthorizedHealth.status).toBe(401);
const unknown = await exports.default.fetch(
new Request("https://diagnostics.test/v1/not-real", { method: "POST" }),
);
expect(unknown.status).toBe(404);
const unauthorized = await exports.default.fetch(
jsonRequest("/v1/events", eventPayload(uuid(1)), "wrong-key"),
);
expect(unauthorized.status).toBe(401);
expect(await unauthorized.json()).toEqual({ error: "unauthorized" });
expect(unauthorized.headers.get("x-request-id")).toMatch(
/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/,
);
const preflight = await exports.default.fetch(
new Request("https://diagnostics.test/v1/events", { method: "OPTIONS" }),
);
expect(preflight.status).toBe(204);
expect(preflight.headers.get("access-control-allow-origin")).toBeNull();
expect(preflight.headers.get("allow")).toBe("POST, OPTIONS");
});
it("applies the source limit before shared-key verification", async () => {
let installLimitCalls = 0;
const limitedEnv: DiagnosticsEnv = {
...env,
SOURCE_RATE_LIMITER: {
limit: async () => ({ success: false }),
} as RateLimit,
INSTALL_RATE_LIMITER: {
limit: async () => {
installLimitCalls += 1;
return { success: true };
},
} as RateLimit,
};
const context = createExecutionContext();
const response = await worker.fetch(
jsonRequest("/v1/events", eventPayload(uuid(3)), "wrong-key", "198.51.100.3"),
limitedEnv,
context,
);
expect(response.status).toBe(429);
expect(response.headers.get("retry-after")).toBe("60");
expect(await response.json()).toEqual({ error: "rate_limited" });
expect(installLimitCalls).toBe(0);
});
it("returns structured errors for invalid bodies and asynchronous storage failures", async () => {
const invalid = await exports.default.fetch(jsonRequest("/v1/events", null));
expect(invalid.status).toBe(400);
expect(await invalid.json()).toEqual({ error: "invalid_body" });
const rejection = new Error("simulated D1 rejection");
const statement = {
bind: () => statement,
first: async () => Promise.reject(rejection),
run: async () => Promise.reject(rejection),
};
const rejectingDatabase = {
prepare: () => statement,
} as unknown as D1Database;
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
const healthContext = createExecutionContext();
const unhealthy = await worker.fetch(
healthRequest(env.INGEST_KEY, "198.51.100.4"),
rejectingEnv,
healthContext,
);
expect(unhealthy.status).toBe(503);
expect(await unhealthy.json()).toEqual({ ok: false, error: "dependency_unavailable" });
const ingestContext = createExecutionContext();
const failedIngest = await worker.fetch(
jsonRequest("/v1/events", eventPayload(uuid(2)), env.INGEST_KEY, "198.51.100.2"),
rejectingEnv,
ingestContext,
);
expect(failedIngest.status).toBe(500);
expect(await failedIngest.json()).toEqual({ error: "internal" });
});
it("deduplicates event batches using the client batch ID", async () => {
const id = uuid(10);
const first = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
const second = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
expect(first.status).toBe(202);
expect(await first.json()).toMatchObject({
ok: true,
id,
stored: 1,
duplicate: false,
});
expect(second.status).toBe(202);
expect(await second.json()).toMatchObject({
ok: true,
id,
stored: 0,
duplicate: true,
});
const row = await env.DB.prepare(
"SELECT event_count AS eventCount, payload_json AS payloadJson FROM event_batches WHERE id = ?",
)
.bind(id)
.first<{ eventCount: number; payloadJson: string }>();
expect(row?.eventCount).toBe(1);
expect(JSON.parse(row?.payloadJson ?? "null")).toEqual([
{
name: "app_open",
timestampMillis: 1,
properties: { screen: "home" },
schemaVersion: 1,
},
]);
});
it("keeps D1 idempotency when the optional analytics index is enabled", async () => {
const points: AnalyticsEngineDataPoint[] = [];
const analytics = {
writeDataPoint: (point: AnalyticsEngineDataPoint) => points.push(point),
} as AnalyticsEngineDataset;
const analyticsEnv: DiagnosticsEnv = { ...env, AE: analytics };
const payload: NormalizedEventsPayload = {
batchId: uuid(11),
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
events: [
{
name: "indexed",
timestampMillis: 1,
properties: {},
schemaVersion: 1,
},
],
};
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: false, stored: 1 });
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: true, stored: 0 });
expect(points).toHaveLength(1);
});
it("keeps the accepted crash blob when a duplicate request arrives", async () => {
const id = uuid(20);
const first = await exports.default.fetch(
jsonRequest("/v1/crashes", crashPayload(id, "first stack")),
);
const second = await exports.default.fetch(
jsonRequest("/v1/crashes", crashPayload(id, "second stack")),
);
expect(first.status).toBe(202);
const firstBody = await first.json<{ fingerprint: string }>();
expect(firstBody).toMatchObject({ ok: true, id, duplicate: false });
expect(second.status).toBe(202);
const secondBody = await second.json<{ fingerprint: string }>();
expect(secondBody).toMatchObject({ ok: true, id, duplicate: true });
const row = await env.DB.prepare(
`SELECT stack_r2_key AS stackKey, breadcrumbs_json AS breadcrumbsJson,
fingerprint
FROM crashes WHERE id = ?`,
)
.bind(id)
.first<{ stackKey: string; breadcrumbsJson: string; fingerprint: string }>();
expect(row?.stackKey).toMatch(new RegExp(`^crashes/${id}/[0-9a-f-]+/stack\\.txt$`));
expect(firstBody.fingerprint).toBe(row?.fingerprint);
expect(secondBody.fingerprint).toBe(row?.fingerprint);
expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([]);
expect(await (await env.BLOBS.get(row?.stackKey ?? "missing"))?.text()).toBe("first stack");
const objects = await env.BLOBS.list({ prefix: `crashes/${id}/` });
expect(objects.objects.map((object) => object.key)).toEqual([row?.stackKey]);
});
it("stores bug metadata as JSON and cleans the duplicate upload attempt", async () => {
const id = uuid(30);
const payload = bugPayload(id, "first logs");
const first = await exports.default.fetch(jsonRequest("/v1/bugs", payload));
const second = await exports.default.fetch(
jsonRequest("/v1/bugs", bugPayload(id, "second logs")),
);
expect(first.status).toBe(202);
expect(await first.json()).toMatchObject({ ok: true, id, duplicate: false });
expect(second.status).toBe(202);
expect(await second.json()).toMatchObject({ ok: true, id, duplicate: true });
const row = await env.DB.prepare(
`SELECT occurred_at AS occurredAt, logs_r2_key AS logsKey,
device_json AS deviceJson, breadcrumbs_json AS breadcrumbsJson
FROM bugs WHERE id = ?`,
)
.bind(id)
.first<{
occurredAt: number;
logsKey: string;
deviceJson: string;
breadcrumbsJson: string;
}>();
expect(row?.occurredAt).toBe(3);
expect(JSON.parse(row?.deviceJson ?? "null")).toEqual({
deviceName: "Test device",
deviceModel: "Model",
operatingSystem: "Test OS",
network: "offline",
batteryLevel: "90%",
});
expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([
{ name: "opened", timestampMillis: 2, properties: {} },
]);
expect(await (await env.BLOBS.get(row?.logsKey ?? "missing"))?.text()).toBe("first logs");
const objects = await env.BLOBS.list({ prefix: `bugs/${id}/` });
expect(objects.objects.map((object) => object.key)).toEqual([row?.logsKey]);
});
it("acknowledges known report IDs without touching an unavailable blob store", async () => {
const crash = normalizedCrash(uuid(31), "accepted stack");
const bug = normalizedBug(uuid(32), "accepted logs");
const firstCrash = await storeCrash(crash, env);
await storeBug(bug, env);
let blobWrites = 0;
const unavailableBlobs = {
put: async () => {
blobWrites += 1;
throw new Error("simulated R2 outage");
},
} as unknown as R2Bucket;
const unavailableEnv: DiagnosticsEnv = { ...env, BLOBS: unavailableBlobs };
await expect(
storeCrash({ ...crash, stackTrace: "retry stack" }, unavailableEnv),
).resolves.toEqual({
id: crash.id,
duplicate: true,
stored: 0,
fingerprint: firstCrash.fingerprint,
});
await expect(
storeBug({ ...bug, logs: "retry logs" }, unavailableEnv),
).resolves.toEqual({ id: bug.id, duplicate: true, stored: 0 });
expect(blobWrites).toBe(0);
});
it("removes uploaded report blobs when D1 rejects the metadata write", async () => {
const rejection = new Error("simulated D1 write rejection");
const rejectingDatabase = databaseWithSession(
() => null,
async () => Promise.reject(rejection),
);
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
const crashId = uuid(33);
const bugId = uuid(34);
const crashResponse = await worker.fetch(
jsonRequest("/v1/crashes", crashPayload(crashId, "orphan candidate"), env.INGEST_KEY, "198.51.100.33"),
rejectingEnv,
createExecutionContext(),
);
const bugResponse = await worker.fetch(
jsonRequest("/v1/bugs", bugPayload(bugId, "orphan candidate"), env.INGEST_KEY, "198.51.100.34"),
rejectingEnv,
createExecutionContext(),
);
expect(crashResponse.status).toBe(500);
expect(await crashResponse.json()).toEqual({ error: "internal" });
expect(bugResponse.status).toBe(500);
expect(await bugResponse.json()).toEqual({ error: "internal" });
expect((await env.BLOBS.list({ prefix: `crashes/${crashId}/` })).objects).toEqual([]);
expect((await env.BLOBS.list({ prefix: `bugs/${bugId}/` })).objects).toEqual([]);
});
it("removes expired rows and their exact R2 objects while preserving current data", async () => {
const oldEventId = uuid(40);
const oldCrashId = uuid(41);
const oldBugId = uuid(42);
const currentEventId = uuid(43);
const oldCrashKey = `crashes/${oldCrashId}/retention/stack.txt`;
const oldBugKey = `bugs/${oldBugId}/retention/logs.txt`;
const oldReceivedAt = Date.now() - 100 * 86_400_000;
await Promise.all([
env.BLOBS.put(oldCrashKey, "expired crash"),
env.BLOBS.put(oldBugKey, "expired logs"),
]);
await env.DB.batch([
env.DB.prepare(
`INSERT INTO event_batches
(id, received_at, install_id, app_version, platform, event_count, payload_json)
VALUES (?, ?, ?, '', '', 1, '[]')`,
).bind(oldEventId, oldReceivedAt, INSTALL_ID),
env.DB.prepare(
`INSERT INTO event_batches
(id, received_at, install_id, app_version, platform, event_count, payload_json)
VALUES (?, ?, ?, '', '', 1, '[]')`,
).bind(currentEventId, Date.now(), INSTALL_ID),
env.DB.prepare(
`INSERT INTO crashes
(id, received_at, occurred_at, install_id, app_version, platform,
exception_type, exception_message, fingerprint, diagnostics_enabled,
stack_r2_key, breadcrumbs_json, schema_version)
VALUES (?, ?, ?, ?, '', '', 'Error', '', 'fingerprint', 1, ?, '[]', 1)`,
).bind(oldCrashId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldCrashKey),
env.DB.prepare(
`INSERT INTO bugs
(id, received_at, occurred_at, install_id, app_version, platform,
what_happened, expected, steps, contact, logs_r2_key,
device_json, breadcrumbs_json, status, schema_version)
VALUES (?, ?, ?, ?, '', '', 'failed', 'worked', '', '', ?, '{}', '[]', 'open', 1)`,
).bind(oldBugId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldBugKey),
]);
await runRetention(env);
for (const [table, id] of [
["event_batches", oldEventId],
["crashes", oldCrashId],
["bugs", oldBugId],
] as const) {
const row = await env.DB.prepare(`SELECT id FROM ${table} WHERE id = ?`).bind(id).first();
expect(row).toBeNull();
}
expect(await env.BLOBS.head(oldCrashKey)).toBeNull();
expect(await env.BLOBS.head(oldBugKey)).toBeNull();
expect(
await env.DB.prepare("SELECT id FROM event_batches WHERE id = ?").bind(currentEventId).first(),
).not.toBeNull();
});
it("bounds a full retention run below the D1 per-invocation query limit", async () => {
let queryCount = 0;
let batchCalls = 0;
const blobDeleteBatchSizes: number[] = [];
const rows = Array.from({ length: 900 }, (_, index) => ({
id: `expired-${index}`,
blobKey: `expired/${index}`,
}));
const database = {
prepare: () => {
const statement = {
bind: () => statement,
all: async () => {
queryCount += 1;
return d1Result(rows, 0);
},
};
return statement;
},
batch: async (statements: D1PreparedStatement[]) => {
batchCalls += 1;
queryCount += statements.length;
if (batchCalls === 9) {
return statements.map(() => d1Result([{ count: 1 }], 0));
}
return statements.map((_, index) => d1Result([], index === 0 ? 1_000 : 900));
},
} as unknown as D1Database;
const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
const blobs = {
delete: async (keys: string | string[]) => {
blobDeleteBatchSizes.push(typeof keys === "string" ? 1 : keys.length);
},
} as unknown as R2Bucket;
try {
await runRetention({ ...env, DB: database, BLOBS: blobs });
} finally {
warning.mockRestore();
}
expect(queryCount).toBe(43);
expect(blobDeleteBatchSizes).toHaveLength(16);
expect(Math.max(...blobDeleteBatchSizes)).toBe(1_000);
});
it("converges an expired report backlog across bounded retention runs", async () => {
const oldReceivedAt = Date.now() - 100 * 86_400_000;
await env.DB.prepare(
`WITH digits(value) AS (
VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9)
), sequence(value) AS (
SELECT thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value + 1
FROM digits AS thousands
CROSS JOIN digits AS hundreds
CROSS JOIN digits AS tens
CROSS JOIN digits AS ones
WHERE thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value < 7201
)
INSERT INTO crashes (
id, received_at, occurred_at, install_id, app_version, platform,
exception_type, exception_message, fingerprint, diagnostics_enabled,
stack_r2_key, breadcrumbs_json, schema_version
)
SELECT 'retention-backlog-' || printf('%04d', value), ?, ?, ?, '', '',
'Error', '', 'fingerprint-' || value, 0, NULL, '[]', 1
FROM sequence`,
)
.bind(oldReceivedAt, oldReceivedAt, INSTALL_ID)
.run();
const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
try {
await runRetention(env);
const afterFirstRun = await env.DB.prepare(
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
).first<{ count: number }>();
expect(afterFirstRun?.count).toBe(1);
await runRetention(env);
const afterSecondRun = await env.DB.prepare(
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
).first<{ count: number }>();
expect(afterSecondRun?.count).toBe(0);
} finally {
warning.mockRestore();
}
});
});
function jsonRequest(
path: string,
body: unknown,
key = env.INGEST_KEY,
source = "198.51.100.1",
): Request {
return new Request(`https://diagnostics.test${path}`, {
method: "POST",
headers: {
"content-type": "application/json; charset=utf-8",
"cf-connecting-ip": source,
"x-vnidrop-install-id": INSTALL_ID,
"x-vnidrop-key": key,
},
body: JSON.stringify(body),
});
}
function healthRequest(key = env.INGEST_KEY, source = "198.51.100.1"): Request {
return new Request("https://diagnostics.test/health", {
headers: {
"cf-connecting-ip": source,
"x-vnidrop-key": key,
},
});
}
function eventPayload(batchId: string): Record<string, unknown> {
return {
batchId,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
events: [
{
name: "app_open",
timestampMillis: 1,
properties: { screen: "home" },
schemaVersion: 1,
},
],
};
}
function crashPayload(id: string, stackTrace: string): Record<string, unknown> {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
exceptionType: "TestError",
exceptionMessage: "failed",
stackTrace,
timestampMillis: 2,
diagnosticsEnabledAtCapture: true,
breadcrumbs: [],
schemaVersion: 1,
};
}
function bugPayload(id: string, logs: string): Record<string, unknown> {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
timestampMillis: 3,
whatHappened: "It failed",
expected: "It should work",
steps: "Open the app",
contact: "",
includeLogs: true,
logs,
device: {
deviceName: "Test device",
deviceModel: "Model",
operatingSystem: "Test OS",
network: "offline",
batteryLevel: "90%",
},
breadcrumbs: [{ name: "opened", timestampMillis: 2, properties: {} }],
schemaVersion: 1,
};
}
function normalizedCrash(id: string, stackTrace: string): NormalizedCrashPayload {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
exceptionType: "TestError",
exceptionMessage: "failed",
stackTrace,
occurredAt: 2,
diagnosticsEnabledAtCapture: true,
breadcrumbs: [],
schemaVersion: 1,
};
}
function normalizedBug(id: string, logs: string): NormalizedBugPayload {
return {
id,
installId: INSTALL_ID,
appVersion: "1.0",
platform: "test",
occurredAt: 3,
whatHappened: "It failed",
expected: "It should work",
steps: "Open the app",
contact: "",
logs,
device: {
deviceName: "Test device",
deviceModel: "Model",
operatingSystem: "Test OS",
network: "offline",
batteryLevel: "90%",
},
breadcrumbs: [],
schemaVersion: 1,
};
}
function databaseWithSession(
first: () => unknown,
run: () => Promise<D1Result>,
): D1Database {
const statement = {
bind: () => statement,
first: async () => first(),
run,
} as unknown as D1PreparedStatement;
const session = {
prepare: () => statement,
} as unknown as D1DatabaseSession;
return {
withSession: () => session,
} as unknown as D1Database;
}
function d1Result<T>(results: T[], changes: number): D1Result<T> {
return { success: true, results, meta: { changes } } as D1Result<T>;
}
function uuid(suffix: number): string {
return `00000000-0000-4000-8000-${suffix.toString().padStart(12, "0")}`;
}

View File

@@ -0,0 +1,14 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ES2022",
"moduleResolution": "Bundler",
"lib": ["ES2022"],
"types": ["./worker-configuration.d.ts"],
"strict": true,
"skipLibCheck": true,
"noEmit": true,
"isolatedModules": true
},
"include": ["worker-configuration.d.ts", "src/**/*.ts"]
}

View File

@@ -0,0 +1,23 @@
import { cloudflareTest, readD1Migrations } from "@cloudflare/vitest-pool-workers";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
const migrationsPath = resolve(dirname(fileURLToPath(import.meta.url)), "migrations");
export default defineConfig({
plugins: [
cloudflareTest(async () => ({
wrangler: { configPath: "./wrangler.jsonc" },
miniflare: {
bindings: {
INGEST_KEY: "test-ingest-key",
TEST_MIGRATIONS: await readD1Migrations(migrationsPath),
},
},
})),
],
test: {
setupFiles: ["./test/apply-migrations.ts"],
},
});

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,57 @@
{
"$schema": "./node_modules/wrangler/config-schema.json",
"name": "vnidrop-diagnostics",
"main": "src/index.ts",
"compatibility_date": "2026-07-14",
"compatibility_flags": ["nodejs_compat"],
"upload_source_maps": true,
"observability": {
"enabled": true,
"head_sampling_rate": 0.1,
},
"triggers": {
"crons": ["17 * * * *"],
},
"d1_databases": [
{
"binding": "DB",
"database_name": "vnidrop-diagnostics",
// Replace this placeholder with the ID returned by `wrangler d1 create`.
"database_id": "00000000-0000-0000-0000-000000000000",
"migrations_dir": "migrations",
},
],
"r2_buckets": [
{
"binding": "BLOBS",
"bucket_name": "vnidrop-diagnostics",
},
],
"ratelimits": [
{
"name": "INSTALL_RATE_LIMITER",
"namespace_id": "1001",
"simple": {
"limit": 30,
"period": 60,
},
},
{
"name": "SOURCE_RATE_LIMITER",
"namespace_id": "1002",
"simple": {
"limit": 120,
"period": 60,
},
},
],
// Optional: bind Analytics Engine as `AE` when event volume justifies it.
// "analytics_engine_datasets": [
// { "binding": "AE", "dataset": "vnidrop_events" },
// ],
"vars": {
"MAX_BODY_BYTES": "262144",
"MAX_EVENTS_PER_BATCH": "50",
"RETENTION_DAYS": "90",
},
}

View File

@@ -37,32 +37,61 @@ plugins {
alias(libs.plugins.kotlinAtomicfu) alias(libs.plugins.kotlinAtomicfu)
} }
// Compile-time switch (gradle.properties or -Pvnidrop.diagnostics.included=false). // Compile-time switches (gradle.properties or -P).
// false: no Share-diagnostics toggle, no telemetry/crash auto-upload stack. // included=false: no Share-diagnostics toggle, no telemetry/crash auto-upload stack.
// endpoint/key both empty: transport is NoOp (safe default until Cloudflare is deployed).
val diagnosticsIncluded: Boolean = val diagnosticsIncluded: Boolean =
(findProperty("vnidrop.diagnostics.included") as String?)?.toBooleanStrictOrNull() ?: true (findProperty("vnidrop.diagnostics.included") as String?)?.toBooleanStrictOrNull() ?: true
val diagnosticsEndpoint: String =
(findProperty("vnidrop.diagnostics.endpoint") as String?)?.trim().orEmpty()
val diagnosticsIngestKey: String =
(findProperty("vnidrop.diagnostics.ingestKey") as String?)?.trim().orEmpty()
check(diagnosticsEndpoint.isEmpty() == diagnosticsIngestKey.isEmpty()) {
"vnidrop.diagnostics.endpoint and vnidrop.diagnostics.ingestKey must be configured together"
}
val diagnosticsBuildConfigDir = layout.buildDirectory.dir("generated/diagnostics/commonMain/kotlin") val diagnosticsBuildConfigDir = layout.buildDirectory.dir("generated/diagnostics/commonMain/kotlin")
val generateDiagnosticsBuildConfig by tasks.registering { val generateDiagnosticsBuildConfig by tasks.registering {
group = "build" group = "build"
description = "Generates DiagnosticsBuildConfig from vnidrop.diagnostics.included" description = "Generates DiagnosticsBuildConfig from vnidrop.diagnostics.* properties"
val outputDir = diagnosticsBuildConfigDir val outputDir = diagnosticsBuildConfigDir
val included = diagnosticsIncluded val included = diagnosticsIncluded
val endpoint = diagnosticsEndpoint
val ingestKey = diagnosticsIngestKey
inputs.property("vnidrop.diagnostics.included", included) inputs.property("vnidrop.diagnostics.included", included)
inputs.property("vnidrop.diagnostics.endpoint", endpoint)
inputs.property("vnidrop.diagnostics.ingestKey", ingestKey)
outputs.dir(outputDir) outputs.dir(outputDir)
doLast { doLast {
val packageDir = outputDir.get().asFile.resolve("com/vnidrop/app/diagnostics") val packageDir = outputDir.get().asFile.resolve("com/vnidrop/app/diagnostics")
packageDir.mkdirs() packageDir.mkdirs()
fun esc(value: String): String = buildString {
for (ch in value) {
when (ch) {
'\\' -> append("\\\\")
'"' -> append("\\\"")
'\n' -> append("\\n")
'\r' -> append("\\r")
'\t' -> append("\\t")
'$' -> append("\\$")
else -> append(ch)
}
}
}
packageDir.resolve("DiagnosticsBuildConfig.kt").writeText( packageDir.resolve("DiagnosticsBuildConfig.kt").writeText(
""" """
|package com.vnidrop.app.diagnostics |package com.vnidrop.app.diagnostics
| |
|/** |/**
| * Generated by shared/build.gradle.kts. | * Generated by shared/build.gradle.kts from vnidrop.diagnostics.* properties.
| * Override with `-Pvnidrop.diagnostics.included=false` or gradle.properties. | *
| * - included=false → no opt-in UI / telemetry / crash hooks
| * - endpoint/key both empty → [NoOpDiagnosticsTransport] (no network)
| */ | */
|object DiagnosticsBuildConfig { |object DiagnosticsBuildConfig {
| const val INCLUDED: Boolean = $included | const val INCLUDED: Boolean = $included
| const val ENDPOINT: String = "${esc(endpoint)}"
| const val INGEST_KEY: String = "${esc(ingestKey)}"
|} |}
| |
""".trimMargin(), """.trimMargin(),

View File

@@ -13,8 +13,16 @@ private class AndroidPendingCrashStore(
@Synchronized @Synchronized
override fun write(report: CrashReport) { override fun write(report: CrashReport) {
if (!isValidDiagnosticId(report.id)) return
directory.mkdirs() directory.mkdirs()
File(directory, "${report.id}.crash").writeText(CrashReportCodec.encode(report), StandardCharsets.UTF_8) val target = File(directory, "${report.id}.crash")
val temporary = File(directory, ".${report.id}.tmp")
val payload = CrashReportCodec.encode(report)
temporary.writeText(payload, StandardCharsets.UTF_8)
if (!temporary.renameTo(target)) {
target.writeText(payload, StandardCharsets.UTF_8)
temporary.delete()
}
} }
@Synchronized @Synchronized
@@ -31,6 +39,34 @@ private class AndroidPendingCrashStore(
@Synchronized @Synchronized
override fun delete(id: String) { override fun delete(id: String) {
if (!isValidDiagnosticId(id)) return
File(directory, "$id.crash").delete() File(directory, "$id.crash").delete()
} }
@Synchronized
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
require(maxCount > 0) { "maxCount must be positive" }
if (!directory.isDirectory) return
directory.listFiles { file -> file.isFile && file.name.endsWith(".tmp") }
.orEmpty()
.forEach(File::delete)
val reports = directory
.listFiles { file -> file.isFile && file.name.endsWith(".crash") }
.orEmpty()
.mapNotNull { file ->
val report = runCatching {
CrashReportCodec.decode(file.readText(StandardCharsets.UTF_8))
}.getOrNull()
if (report == null) {
file.delete()
null
} else {
file to report
}
}
.sortedByDescending { (_, report) -> report.timestampMillis }
reports.forEachIndexed { index, (file, report) ->
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) file.delete()
}
}
} }

View File

@@ -0,0 +1,37 @@
package com.vnidrop.app.diagnostics
import java.io.BufferedReader
import java.io.InputStreamReader
import java.net.HttpURLConnection
import java.net.URI
import java.nio.charset.StandardCharsets
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
actual suspend fun platformHttpPost(
url: String,
headers: Map<String, String>,
bodyUtf8: String,
): PlatformHttpResponse = withContext(Dispatchers.IO) {
val connection = (URI(url).toURL().openConnection() as HttpURLConnection).apply {
requestMethod = "POST"
doOutput = true
connectTimeout = 15_000
readTimeout = 30_000
setRequestProperty("Content-Type", "application/json; charset=utf-8")
headers.forEach { (key, value) -> setRequestProperty(key, value) }
}
try {
connection.outputStream.use { output ->
output.write(bodyUtf8.toByteArray(StandardCharsets.UTF_8))
}
val code = connection.responseCode
val stream = if (code in 200..299) connection.inputStream else connection.errorStream
val body = stream?.use { input ->
BufferedReader(InputStreamReader(input, StandardCharsets.UTF_8)).readText()
}.orEmpty()
PlatformHttpResponse(code, body)
} finally {
connection.disconnect()
}
}

View File

@@ -3,7 +3,7 @@ package com.vnidrop.app
import com.vnidrop.app.core.CoreGateway import com.vnidrop.app.core.CoreGateway
import com.vnidrop.app.core.CoreRepository import com.vnidrop.app.core.CoreRepository
import com.vnidrop.app.diagnostics.DiagnosticsCoordinator import com.vnidrop.app.diagnostics.DiagnosticsCoordinator
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport import com.vnidrop.app.diagnostics.createDiagnosticsTransport
import com.vnidrop.app.feature.approvals.ApprovalCoordinator import com.vnidrop.app.feature.approvals.ApprovalCoordinator
import com.vnidrop.app.feature.send.AppFilePreviewRepository import com.vnidrop.app.feature.send.AppFilePreviewRepository
import com.vnidrop.app.feature.send.createPlatformPreviewStore import com.vnidrop.app.feature.send.createPlatformPreviewStore
@@ -18,6 +18,9 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel import kotlinx.coroutines.cancel
import kotlinx.coroutines.flow.drop
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.launch
class AppGraph( class AppGraph(
val dependencies: AppDependencies, val dependencies: AppDependencies,
@@ -45,7 +48,11 @@ class AppGraph(
platform = dependencies.environment.name, platform = dependencies.environment.name,
preferencesRepository = preferencesRepository, preferencesRepository = preferencesRepository,
scope = applicationScope, scope = applicationScope,
transport = NoOpDiagnosticsTransport(), transport = createDiagnosticsTransport(
appVersion = dependencies.environment.appVersion,
platform = dependencies.environment.name,
installIdProvider = { preferencesRepository.ensureDiagnosticsInstallId() },
),
) )
val approvalCoordinator = ApprovalCoordinator( val approvalCoordinator = ApprovalCoordinator(
repository = coreRepository, repository = coreRepository,
@@ -59,6 +66,12 @@ class AppGraph(
init { init {
AppLogger.initialize(dependencies.environment.defaultCoreDataDir) AppLogger.initialize(dependencies.environment.defaultCoreDataDir)
diagnostics.start() diagnostics.start()
applicationScope.launch {
visibility.isForeground
.drop(1)
.filter { isForeground -> !isForeground }
.collect { diagnostics.telemetry.flush() }
}
} }
fun close() { fun close() {

View File

@@ -1,6 +1,8 @@
package com.vnidrop.app.diagnostics package com.vnidrop.app.diagnostics
import com.vnidrop.app.logging.platformNowMillis import com.vnidrop.app.logging.platformNowMillis
import kotlin.concurrent.atomics.AtomicReference
import kotlin.concurrent.atomics.ExperimentalAtomicApi
/** /**
* Fixed-size ring of high-level app breadcrumbs for crash / bug context. * Fixed-size ring of high-level app breadcrumbs for crash / bug context.
@@ -9,6 +11,7 @@ import com.vnidrop.app.logging.platformNowMillis
* Updates are best-effort under concurrency; losing a breadcrumb is preferable * Updates are best-effort under concurrency; losing a breadcrumb is preferable
* to blocking a dying process on a lock. * to blocking a dying process on a lock.
*/ */
@OptIn(ExperimentalAtomicApi::class)
class BreadcrumbBuffer( class BreadcrumbBuffer(
private val capacity: Int = DefaultCapacity, private val capacity: Int = DefaultCapacity,
) { ) {
@@ -16,28 +19,29 @@ class BreadcrumbBuffer(
require(capacity > 0) { "capacity must be positive" } require(capacity > 0) { "capacity must be positive" }
} }
@Volatile private val items = AtomicReference<List<Breadcrumb>>(emptyList())
private var items: List<Breadcrumb> = emptyList()
fun add(name: String, properties: Map<String, String> = emptyMap(), timestampMillis: Long = platformNowMillis()) { fun add(name: String, properties: Map<String, String> = emptyMap(), timestampMillis: Long = platformNowMillis()) {
val sanitizedName = sanitizeDiagnosticName(name)
if (sanitizedName.isBlank()) return
val crumb = Breadcrumb( val crumb = Breadcrumb(
name = name.take(MaxNameLength), name = sanitizedName,
timestampMillis = timestampMillis, timestampMillis = timestampMillis,
properties = LogRedactor.redactMap(properties).mapValues { it.value.take(MaxPropertyValueLength) }, properties = sanitizeDiagnosticProperties(properties),
) )
val current = items while (true) {
items = (current + crumb).takeLast(capacity) val current = items.load()
if (items.compareAndSet(current, (current + crumb).takeLast(capacity))) return
}
} }
fun snapshot(): List<Breadcrumb> = items fun snapshot(): List<Breadcrumb> = items.load()
fun clear() { fun clear() {
items = emptyList() items.store(emptyList())
} }
companion object { companion object {
const val DefaultCapacity = 40 const val DefaultCapacity = 40
private const val MaxNameLength = 64
private const val MaxPropertyValueLength = 128
} }
} }

View File

@@ -5,6 +5,7 @@ import com.vnidrop.app.logging.AppLogger
import com.vnidrop.app.logging.platformNowMillis import com.vnidrop.app.logging.platformNowMillis
import com.vnidrop.app.preferences.PreferencesRepository import com.vnidrop.app.preferences.PreferencesRepository
import com.vnidrop.app.util.randomUuidString import com.vnidrop.app.util.randomUuidString
import kotlinx.coroutines.CancellationException
data class BugReportDraft( data class BugReportDraft(
val whatHappened: String, val whatHappened: String,
@@ -24,7 +25,7 @@ class BugReportService(
private val appVersion: String, private val appVersion: String,
private val platform: String, private val platform: String,
private val logReader: () -> String = { private val logReader: () -> String = {
LogRedactor.redact(AppLogger.readLatestLogs(AppLogger.DefaultBugReportLogBytes)) AppLogger.readLatestLogs(AppLogger.DefaultBugReportLogBytes)
}, },
) { ) {
fun assemble( fun assemble(
@@ -32,25 +33,25 @@ class BugReportService(
deviceInfo: DeviceInfo?, deviceInfo: DeviceInfo?,
installId: String, installId: String,
): BugReport { ): BugReport {
val logs = if (draft.includeLogs) logReader() else "" val logs = if (draft.includeLogs) readReportLogs() else ""
return BugReport( return BugReport(
id = randomUuidString(), id = randomUuidString(),
timestampMillis = platformNowMillis(), timestampMillis = platformNowMillis(),
installId = installId, installId = sanitizeDiagnosticsInstallId(installId),
appVersion = appVersion, appVersion = appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
platform = platform, platform = platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
whatHappened = draft.whatHappened.trim().take(MaxFieldLength), whatHappened = draft.whatHappened.trim().takeUtf8Bytes(MaxFieldBytes),
expected = draft.expected.trim().take(MaxFieldLength), expected = draft.expected.trim().takeUtf8Bytes(MaxFieldBytes),
steps = draft.steps.trim().take(MaxFieldLength), steps = draft.steps.trim().takeUtf8Bytes(MaxFieldBytes),
contact = draft.contact.trim().take(MaxContactLength), contact = draft.contact.trim().takeUtf8Bytes(MaxContactBytes),
includeLogs = draft.includeLogs, includeLogs = draft.includeLogs,
logs = logs, logs = logs,
device = DeviceSnapshot( device = DeviceSnapshot(
deviceName = deviceInfo?.deviceName, deviceName = deviceInfo?.deviceName?.takeUtf8Bytes(128),
deviceModel = deviceInfo?.deviceModel, deviceModel = deviceInfo?.deviceModel?.takeUtf8Bytes(128),
operatingSystem = deviceInfo?.operatingSystem ?: platform, operatingSystem = (deviceInfo?.operatingSystem ?: platform).takeUtf8Bytes(192),
network = deviceInfo?.network, network = deviceInfo?.network?.takeUtf8Bytes(96),
batteryLevel = deviceInfo?.batteryLevel, batteryLevel = deviceInfo?.batteryLevel?.takeUtf8Bytes(64),
), ),
breadcrumbs = breadcrumbs.snapshot(), breadcrumbs = breadcrumbs.snapshot(),
) )
@@ -67,7 +68,13 @@ class BugReportService(
} }
val installId = preferencesRepository.ensureDiagnosticsInstallId() val installId = preferencesRepository.ensureDiagnosticsInstallId()
val report = assemble(draft, deviceInfo, installId) val report = assemble(draft, deviceInfo, installId)
val send = transport.sendBugReport(report) val send = try {
transport.sendBugReport(report)
} catch (cancelled: CancellationException) {
throw cancelled
} catch (error: Throwable) {
Result.failure(error)
}
return send.fold( return send.fold(
onSuccess = { onSuccess = {
AppLogger.info("bug_report", "submitted", mapOf("id" to report.id)) AppLogger.info("bug_report", "submitted", mapOf("id" to report.id))
@@ -80,10 +87,14 @@ class BugReportService(
) )
} }
fun previewLogBytes(): Int = logReader().encodeToByteArray().size fun previewLogBytes(): Int = readReportLogs().encodeToByteArray().size
private fun readReportLogs(): String =
LogRedactor.redact(logReader()).takeUtf8Bytes(MaxLogBytes)
companion object { companion object {
private const val MaxFieldLength = 4_000 internal const val MaxLogBytes = 192 * 1024
private const val MaxContactLength = 320 private const val MaxFieldBytes = 4_000
private const val MaxContactBytes = 320
} }
} }

View File

@@ -5,13 +5,18 @@ import com.vnidrop.app.logging.platformNowMillis
import com.vnidrop.app.preferences.PreferencesRepository import com.vnidrop.app.preferences.PreferencesRepository
import com.vnidrop.app.util.randomUuidString import com.vnidrop.app.util.randomUuidString
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.first
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlin.concurrent.atomics.AtomicBoolean
import kotlin.concurrent.atomics.AtomicReference
import kotlin.concurrent.atomics.ExperimentalAtomicApi
/** /**
* Captures uncaught exceptions to disk, then uploads on a later launch when * Captures uncaught exceptions to disk, then uploads on a later launch when
* diagnostics is enabled (and when a real [DiagnosticsTransport] is wired). * diagnostics is enabled (and when a real [DiagnosticsTransport] is wired).
*/ */
@OptIn(ExperimentalAtomicApi::class)
class CrashReporter( class CrashReporter(
private val store: PendingCrashStore, private val store: PendingCrashStore,
private val preferencesRepository: PreferencesRepository, private val preferencesRepository: PreferencesRepository,
@@ -21,69 +26,133 @@ class CrashReporter(
private val platform: String, private val platform: String,
private val scope: CoroutineScope, private val scope: CoroutineScope,
) { ) {
@Volatile private var installed = false private val installed = AtomicBoolean(false)
@Volatile private var lastInstallId: String = "" private val observingPreferences = AtomicBoolean(false)
@Volatile private var lastDiagnosticsEnabled: Boolean = false private val capturePolicy = AtomicReference(CrashCapturePolicy())
fun startObservingPreferences() { fun startObservingPreferences() {
if (!observingPreferences.compareAndSet(false, true)) return
scope.launch { scope.launch {
preferencesRepository.preferences.collect { prefs -> preferencesRepository.preferences.collect { prefs ->
lastInstallId = prefs.diagnosticsInstallId capturePolicy.store(
lastDiagnosticsEnabled = prefs.diagnosticsEnabled CrashCapturePolicy(
installId = prefs.diagnosticsInstallId,
diagnosticsEnabled = prefs.diagnosticsEnabled,
),
)
if (!prefs.diagnosticsEnabled) {
runCatching(::deleteAllPending)
}
} }
} }
} }
fun installUnhandledExceptionHandler() { fun installUnhandledExceptionHandler() {
if (!DiagnosticsBuildConfig.INCLUDED) return if (!DiagnosticsBuildConfig.INCLUDED) return
if (installed) return if (!installed.compareAndSet(false, true)) return
installed = true
installPlatformCrashHook { throwable -> installPlatformCrashHook { throwable ->
capture(throwable) capture(throwable)
} }
} }
fun capture(throwable: Throwable, diagnosticsEnabledOverride: Boolean? = null): CrashReport { fun capture(throwable: Throwable, diagnosticsEnabledOverride: Boolean? = null): CrashReport {
val policy = capturePolicy.load()
val report = CrashReport( val report = CrashReport(
id = randomUuidString(), id = randomUuidString(),
timestampMillis = platformNowMillis(), timestampMillis = platformNowMillis(),
installId = lastInstallId, installId = sanitizeDiagnosticsInstallId(policy.installId),
appVersion = appVersion, appVersion = appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
platform = platform, platform = platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
exceptionType = throwable::class.simpleName ?: "Throwable", exceptionType = throwable::class.simpleName ?: "Throwable",
exceptionMessage = LogRedactor.redact(throwable.message.orEmpty()).take(MaxMessageLength), exceptionMessage = LogRedactor.redact(throwable.message.orEmpty()).takeUtf8Bytes(MaxMessageBytes),
stackTrace = LogRedactor.redact(throwable.stackTraceToString()).take(MaxStackLength), stackTrace = LogRedactor.redact(throwable.stackTraceToString()).takeUtf8Bytes(MaxStackBytes),
breadcrumbs = breadcrumbs.snapshot(), breadcrumbs = breadcrumbs.snapshot(),
diagnosticsEnabledAtCapture = diagnosticsEnabledOverride ?: lastDiagnosticsEnabled, diagnosticsEnabledAtCapture = diagnosticsEnabledOverride ?: policy.diagnosticsEnabled,
) )
runCatching { store.write(report) } if (report.diagnosticsEnabledAtCapture != false) {
runCatching { store.write(report) }
runCatching {
store.prune(
olderThanTimestampMillis = platformNowMillis() - LocalRetentionMillis,
maxCount = MaxLocalCrashCount,
)
}
}
AppLogger.error("crash", "captured crash ${report.id}", throwable) AppLogger.error("crash", "captured crash ${report.id}", throwable)
return report return report
} }
/** /**
* Uploads pending crashes that were captured with diagnostics enabled. * Uploads pending crashes that were captured with diagnostics enabled.
* Local files are always kept for bug-report attachment until deleted after successful send. * Local files are deleted after successful delivery or bounded by local retention.
*/ */
suspend fun flushPending() { suspend fun flushPending() {
val diagnosticsEnabled = preferencesRepository.preferences.first().diagnosticsEnabled store.prune(
if (!diagnosticsEnabled) return olderThanTimestampMillis = platformNowMillis() - LocalRetentionMillis,
maxCount = MaxLocalCrashCount,
)
for (report in store.list()) { for (report in store.list()) {
// Only auto-upload crashes captured while diagnostics was on. val preferences = preferencesRepository.preferences.first()
if (!report.diagnosticsEnabledAtCapture) continue if (!preferences.diagnosticsEnabled || capturePolicy.load().diagnosticsEnabled == false) {
val result = transport.sendCrash(report) deleteAllPending()
if (result.isSuccess) { return
store.delete(report.id)
} }
if (report.diagnosticsEnabledAtCapture == false) {
store.delete(report.id)
continue
}
val installId = sanitizeDiagnosticsInstallId(
preferences.diagnosticsInstallId.ifBlank {
preferencesRepository.ensureDiagnosticsInstallId()
},
)
val resolved = report.copy(
installId = report.installId.ifBlank { installId },
diagnosticsEnabledAtCapture = true,
)
if (resolved != report) store.write(resolved)
if (
!preferencesRepository.preferences.first().diagnosticsEnabled ||
capturePolicy.load().diagnosticsEnabled == false
) {
deleteAllPending()
return
}
val result = try {
transport.sendCrash(resolved)
} catch (cancelled: CancellationException) {
throw cancelled
} catch (error: Throwable) {
Result.failure(error)
}
if (result.isSuccess) {
store.delete(resolved.id)
continue
}
val error = result.exceptionOrNull()
if (error?.isPermanentDiagnosticsPayloadRejection() == true) {
store.delete(resolved.id)
continue
}
break
} }
} }
fun latestLocalCrash(): CrashReport? = store.list().maxByOrNull { it.timestampMillis } private fun deleteAllPending() {
store.list().forEach { report -> store.delete(report.id) }
}
companion object { companion object {
private const val MaxMessageLength = 2_000 private const val MaxMessageBytes = 2_000
private const val MaxStackLength = 32_000 private const val MaxStackBytes = 32_000
private const val MaxLocalCrashCount = 20
private const val LocalRetentionMillis = 30L * 86_400_000L
} }
} }
private data class CrashCapturePolicy(
val installId: String = "",
val diagnosticsEnabled: Boolean? = null,
)
expect fun installPlatformCrashHook(onCrash: (Throwable) -> Unit) expect fun installPlatformCrashHook(onCrash: (Throwable) -> Unit)

View File

@@ -0,0 +1,221 @@
package com.vnidrop.app.diagnostics
/**
* Minimal JSON encoding for diagnostics payloads (no kotlinx.serialization dependency).
*/
internal object DiagnosticsJson {
internal const val MaxRequestBytes = 256 * 1024
internal const val MaxInstallIdBytes = 80
internal const val MaxAppVersionBytes = 40
internal const val MaxPlatformBytes = 40
private const val MaxBreadcrumbsJsonBytes = 16_000
private const val MaxBreadcrumbs = 40
private const val SizedBatchId = "00000000-0000-4000-8000-000000000000"
fun eventsBody(
batchId: String,
installId: String,
appVersion: String,
platform: String,
events: List<TelemetryEvent>,
): String = buildString {
append('{')
appendJsonField("batchId", batchId)
append(',')
appendJsonField("installId", installId)
append(',')
appendJsonField("appVersion", appVersion)
append(',')
appendJsonField("platform", platform)
append(',')
append("\"events\":[")
events.forEachIndexed { index, event ->
if (index > 0) append(',')
append('{')
appendJsonField("name", event.name)
append(',')
append("\"timestampMillis\":")
append(event.timestampMillis)
append(',')
append("\"schemaVersion\":")
append(event.schemaVersion)
append(',')
append("\"properties\":")
appendStringMap(event.properties)
append('}')
}
append("]}")
}
fun eventBatchFitsRequest(events: List<TelemetryEvent>): Boolean =
eventsBody(
batchId = SizedBatchId,
installId = "\u0000".repeat(MaxInstallIdBytes),
appVersion = "\u0000".repeat(MaxAppVersionBytes),
platform = "\u0000".repeat(MaxPlatformBytes),
events = events,
).encodeToByteArray().size <= MaxRequestBytes
fun crashBody(report: CrashReport): String = buildString {
append('{')
appendJsonField("id", report.id)
append(',')
append("\"timestampMillis\":")
append(report.timestampMillis)
append(',')
appendJsonField("installId", report.installId)
append(',')
appendJsonField("appVersion", report.appVersion)
append(',')
appendJsonField("platform", report.platform)
append(',')
appendJsonField("exceptionType", report.exceptionType)
append(',')
appendJsonField("exceptionMessage", report.exceptionMessage)
append(',')
appendJsonField("stackTrace", report.stackTrace)
append(',')
append("\"diagnosticsEnabledAtCapture\":")
append(requireNotNull(report.diagnosticsEnabledAtCapture) {
"crash consent must be resolved before delivery"
})
append(',')
append("\"schemaVersion\":")
append(report.schemaVersion)
append(',')
append("\"breadcrumbs\":")
appendBreadcrumbs(report.breadcrumbs)
append('}')
}
fun bugBody(report: BugReport): String {
val logs = if (report.includeLogs) report.logs else ""
val complete = buildBugBody(report, logs)
if (complete.encodeToByteArray().size <= MaxRequestBytes || logs.isEmpty()) return complete
var best = buildBugBody(report, "")
if (best.encodeToByteArray().size > MaxRequestBytes) return best
var minimumBytes = 0
var maximumBytes = logs.encodeToByteArray().size
while (minimumBytes <= maximumBytes) {
val candidateBytes = minimumBytes + (maximumBytes - minimumBytes) / 2
val candidate = buildBugBody(report, logs.takeUtf8Bytes(candidateBytes))
if (candidate.encodeToByteArray().size <= MaxRequestBytes) {
best = candidate
minimumBytes = candidateBytes + 1
} else {
maximumBytes = candidateBytes - 1
}
}
return best
}
private fun buildBugBody(report: BugReport, logs: String): String = buildString {
append('{')
appendJsonField("id", report.id)
append(',')
append("\"timestampMillis\":")
append(report.timestampMillis)
append(',')
appendJsonField("installId", report.installId)
append(',')
appendJsonField("appVersion", report.appVersion)
append(',')
appendJsonField("platform", report.platform)
append(',')
appendJsonField("whatHappened", report.whatHappened)
append(',')
appendJsonField("expected", report.expected)
append(',')
appendJsonField("steps", report.steps)
append(',')
appendJsonField("contact", report.contact)
append(',')
append("\"includeLogs\":")
append(report.includeLogs)
append(',')
appendJsonField("logs", logs)
append(',')
append("\"schemaVersion\":")
append(report.schemaVersion)
append(',')
append("\"device\":{")
appendJsonField("deviceName", report.device.deviceName.orEmpty())
append(',')
appendJsonField("deviceModel", report.device.deviceModel.orEmpty())
append(',')
appendJsonField("operatingSystem", report.device.operatingSystem)
append(',')
appendJsonField("network", report.device.network.orEmpty())
append(',')
appendJsonField("batteryLevel", report.device.batteryLevel.orEmpty())
append("},")
append("\"breadcrumbs\":")
appendBreadcrumbs(report.breadcrumbs)
append('}')
}
private fun StringBuilder.appendBreadcrumbs(crumbs: List<Breadcrumb>) {
append('[')
var encodedBytes = 2
var appended = 0
for (crumb in crumbs) {
if (appended == MaxBreadcrumbs) break
val name = sanitizeDiagnosticName(crumb.name)
if (name.isBlank() || crumb.timestampMillis < 0) continue
val encoded = buildString {
append('{')
appendJsonField("name", name)
append(',')
append("\"timestampMillis\":")
append(crumb.timestampMillis)
append(',')
append("\"properties\":")
appendStringMap(crumb.properties)
append('}')
}
val additionBytes = encoded.encodeToByteArray().size + if (appended == 0) 0 else 1
if (encodedBytes + additionBytes > MaxBreadcrumbsJsonBytes) break
if (appended > 0) append(',')
append(encoded)
encodedBytes += additionBytes
appended += 1
}
append(']')
}
private fun StringBuilder.appendStringMap(map: Map<String, String>) {
append('{')
sanitizeDiagnosticProperties(map).entries.forEachIndexed { index, (key, value) ->
if (index > 0) append(',')
appendJsonField(key, value)
}
append('}')
}
private fun StringBuilder.appendJsonField(key: String, value: String) {
append('"')
append(escape(key))
append("\":\"")
append(escape(value))
append('"')
}
internal fun escape(raw: String): String = buildString(raw.length + 8) {
for (ch in raw) {
when (ch) {
'\\' -> append("\\\\")
'"' -> append("\\\"")
'\n' -> append("\\n")
'\r' -> append("\\r")
'\t' -> append("\\t")
else -> if (ch.code < 0x20) {
append("\\u")
append(ch.code.toString(16).padStart(4, '0'))
} else {
append(ch)
}
}
}
}
}

View File

@@ -12,6 +12,12 @@ data class TelemetryEvent(
val schemaVersion: Int = DiagnosticsSchemaVersion, val schemaVersion: Int = DiagnosticsSchemaVersion,
) )
/** One idempotent upload unit; [id] remains stable when delivery is retried. */
data class TelemetryBatch(
val id: String,
val events: List<TelemetryEvent>,
)
data class Breadcrumb( data class Breadcrumb(
val name: String, val name: String,
val timestampMillis: Long, val timestampMillis: Long,
@@ -28,8 +34,8 @@ data class CrashReport(
val exceptionMessage: String, val exceptionMessage: String,
val stackTrace: String, val stackTrace: String,
val breadcrumbs: List<Breadcrumb>, val breadcrumbs: List<Breadcrumb>,
/** Whether diagnostics was opted in when the crash was captured. */ /** `null` only while a startup crash is waiting for the persisted preference to load. */
val diagnosticsEnabledAtCapture: Boolean, val diagnosticsEnabledAtCapture: Boolean?,
val schemaVersion: Int = DiagnosticsSchemaVersion, val schemaVersion: Int = DiagnosticsSchemaVersion,
) )

View File

@@ -0,0 +1,39 @@
package com.vnidrop.app.diagnostics
internal const val MaxDiagnosticProperties = 12
internal const val MaxDiagnosticPropertyKeyBytes = 40
internal const val MaxDiagnosticPropertyValueBytes = 128
internal const val MaxDiagnosticNameBytes = 64
internal fun sanitizeDiagnosticsInstallId(value: String): String {
val trimmed = value.trim()
if (trimmed.any { it.code < 0x20 || it.code == 0x7f }) return ""
return trimmed.takeUtf8Bytes(DiagnosticsJson.MaxInstallIdBytes)
}
internal fun sanitizeDiagnosticName(name: String): String =
name.takeUtf8Bytes(MaxDiagnosticNameBytes)
internal fun sanitizeDiagnosticProperties(properties: Map<String, String>): Map<String, String> {
val sanitized = LinkedHashMap<String, String>(minOf(properties.size, MaxDiagnosticProperties))
for ((rawKey, rawValue) in properties) {
val key = rawKey.takeUtf8Bytes(MaxDiagnosticPropertyKeyBytes)
if (key.isEmpty() || key in sanitized) continue
sanitized[key] = LogRedactor.redact(rawValue).takeUtf8Bytes(MaxDiagnosticPropertyValueBytes)
if (sanitized.size == MaxDiagnosticProperties) break
}
return sanitized
}
internal fun String.takeUtf8Bytes(maxBytes: Int): String {
require(maxBytes >= 0) { "maxBytes must not be negative" }
val encoded = encodeToByteArray()
if (encoded.size <= maxBytes) return this
for (endIndex in maxBytes downTo (maxBytes - 3).coerceAtLeast(0)) {
val decoded = runCatching {
encoded.decodeToString(0, endIndex, throwOnInvalidSequence = true)
}.getOrNull()
if (decoded != null) return decoded
}
return ""
}

View File

@@ -1,35 +1,42 @@
package com.vnidrop.app.diagnostics package com.vnidrop.app.diagnostics
/** /** Network boundary for diagnostics; keep batching and validation client-side. */
* Network boundary for diagnostics. Production will swap [NoOpDiagnosticsTransport]
* for a Cloudflare Worker client; keep batching/validation client-side.
*/
interface DiagnosticsTransport { interface DiagnosticsTransport {
suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit> suspend fun sendEvents(batch: TelemetryBatch): Result<Unit>
suspend fun sendCrash(report: CrashReport): Result<Unit> suspend fun sendCrash(report: CrashReport): Result<Unit>
suspend fun sendBugReport(report: BugReport): Result<Unit> suspend fun sendBugReport(report: BugReport): Result<Unit>
} }
/** Accepts payloads without leaving the device. Used until Cloudflare is wired. */ internal class DiagnosticsUnavailableException : IllegalStateException("diagnostics delivery is not configured")
internal fun Throwable.isPermanentDiagnosticsPayloadRejection(): Boolean =
this is DiagnosticsPayloadException ||
(this is DiagnosticsHttpException && statusCode in setOf(400, 413, 415, 422))
/** Fails delivery without leaving the device. Used until a remote endpoint is configured. */
class NoOpDiagnosticsTransport : DiagnosticsTransport { class NoOpDiagnosticsTransport : DiagnosticsTransport {
override suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit> = Result.success(Unit) override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> = unavailable()
override suspend fun sendCrash(report: CrashReport): Result<Unit> = Result.success(Unit) override suspend fun sendCrash(report: CrashReport): Result<Unit> = unavailable()
override suspend fun sendBugReport(report: BugReport): Result<Unit> = Result.success(Unit) override suspend fun sendBugReport(report: BugReport): Result<Unit> = unavailable()
private fun unavailable(): Result<Unit> = Result.failure(DiagnosticsUnavailableException())
} }
/** /**
* Test double that records calls and can fail on demand. * Test double that records calls and can fail on demand.
*/ */
class RecordingDiagnosticsTransport : DiagnosticsTransport { class RecordingDiagnosticsTransport : DiagnosticsTransport {
val events = mutableListOf<List<TelemetryEvent>>() val eventBatches = mutableListOf<TelemetryBatch>()
val events: List<List<TelemetryEvent>>
get() = eventBatches.map(TelemetryBatch::events)
val crashes = mutableListOf<CrashReport>() val crashes = mutableListOf<CrashReport>()
val bugReports = mutableListOf<BugReport>() val bugReports = mutableListOf<BugReport>()
var eventsResult: Result<Unit> = Result.success(Unit) var eventsResult: Result<Unit> = Result.success(Unit)
var crashResult: Result<Unit> = Result.success(Unit) var crashResult: Result<Unit> = Result.success(Unit)
var bugResult: Result<Unit> = Result.success(Unit) var bugResult: Result<Unit> = Result.success(Unit)
override suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit> { override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> {
this.events += events eventBatches += batch
return eventsResult return eventsResult
} }

View File

@@ -0,0 +1,192 @@
package com.vnidrop.app.diagnostics
import com.vnidrop.app.logging.AppLogger
import kotlinx.coroutines.CancellationException
/**
* HTTPS client for the Cloudflare diagnostics Worker.
* No-ops are preferred when [baseUrl] is blank — see [createDiagnosticsTransport].
*/
class HttpDiagnosticsTransport(
baseUrl: String,
private val ingestKey: String,
private val appVersion: String = "",
private val platform: String = "",
private val installIdProvider: suspend () -> String = { "" },
private val post: suspend (url: String, headers: Map<String, String>, body: String) -> PlatformHttpResponse =
{ url, headers, body -> platformHttpPost(url, headers, body) },
) : DiagnosticsTransport {
private val root = baseUrl.trim().trimEnd('/')
init {
require(root.isEmpty() || root.isAllowedDiagnosticsEndpoint()) {
"diagnostics endpoint must use HTTPS unless it targets a loopback host"
}
require(root.isEmpty() || ingestKey.isNotBlank()) {
"diagnostics ingest key must be configured when the endpoint is set"
}
}
override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> {
if (batch.events.isEmpty()) return Result.success(Unit)
if (batch.events.size > TelemetryRecorder.MaxEventsPerBatch) {
return Result.failure(DiagnosticsPayloadException("diagnostics event batch is too large"))
}
if (batch.events.any { it.name.isBlank() || it.timestampMillis < 0 }) {
return Result.failure(DiagnosticsPayloadException("diagnostics event batch is invalid"))
}
val installId = sanitizeDiagnosticsInstallId(installIdProvider())
val body = DiagnosticsJson.eventsBody(
batch.id,
installId,
appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
batch.events,
)
return postJson("/v1/events", body, installId, batch.id)
}
override suspend fun sendCrash(report: CrashReport): Result<Unit> {
if (report.diagnosticsEnabledAtCapture == null) {
return Result.failure(DiagnosticsPayloadException("crash consent is unresolved"))
}
val body = DiagnosticsJson.crashBody(report)
return postJson("/v1/crashes", body, report.installId, report.id)
}
override suspend fun sendBugReport(report: BugReport): Result<Unit> {
val body = DiagnosticsJson.bugBody(report)
return postJson("/v1/bugs", body, report.installId, report.id)
}
private suspend fun postJson(
path: String,
body: String,
installId: String,
expectedId: String,
): Result<Unit> {
if (root.isEmpty()) {
return Result.failure(IllegalStateException("diagnostics endpoint is not configured"))
}
if (body.encodeToByteArray().size > DiagnosticsJson.MaxRequestBytes) {
return Result.failure(DiagnosticsPayloadException("diagnostics $path payload is too large"))
}
return try {
val response = post(
"$root$path",
mapOf(
"X-VniDrop-Key" to ingestKey,
"X-VniDrop-Install-Id" to installId,
"Accept" to "application/json",
),
body,
)
if (response.statusCode !in 200..299) {
AppLogger.warn(
"diagnostics",
"transport rejected $path",
mapOf("status" to response.statusCode.toString()),
)
throw DiagnosticsHttpException(response.statusCode, path)
}
if (!response.body.isSuccessfulDiagnosticsAcknowledgement(expectedId)) {
throw DiagnosticsProtocolException(path)
}
Result.success(Unit)
} catch (cancelled: CancellationException) {
throw cancelled
} catch (error: Throwable) {
Result.failure(error)
}
}
}
internal class DiagnosticsHttpException(
val statusCode: Int,
path: String,
) : IllegalStateException("diagnostics $path failed: HTTP $statusCode")
internal class DiagnosticsProtocolException(path: String) :
IllegalStateException("diagnostics $path returned an invalid acknowledgement")
internal class DiagnosticsPayloadException(message: String) : IllegalArgumentException(message)
/**
* Builds transport from compile-time config. Empty endpoint and key → [NoOpDiagnosticsTransport].
*/
fun createDiagnosticsTransport(
appVersion: String,
platform: String,
installIdProvider: suspend () -> String,
): DiagnosticsTransport {
val endpoint = DiagnosticsBuildConfig.ENDPOINT.trim()
val ingestKey = DiagnosticsBuildConfig.INGEST_KEY.trim()
if (endpoint.isEmpty() && ingestKey.isEmpty()) return NoOpDiagnosticsTransport()
check(endpoint.isNotEmpty() && ingestKey.isNotEmpty()) {
"diagnostics endpoint and ingest key must be configured together"
}
return HttpDiagnosticsTransport(
baseUrl = endpoint,
ingestKey = ingestKey,
appVersion = appVersion,
platform = platform,
installIdProvider = installIdProvider,
)
}
private fun String.isSuccessfulDiagnosticsAcknowledgement(expectedId: String): Boolean {
val json = trim()
if (!SuccessfulAcknowledgement.matches(json)) return false
if (AcknowledgementOk.findAll(json).count() != 1) return false
val ids = AcknowledgementId.findAll(json).toList()
return ids.size == 1 &&
ids.single().groupValues[1] == "\"${DiagnosticsJson.escape(expectedId.lowercase())}\""
}
private fun String.isAllowedDiagnosticsEndpoint(): Boolean {
if (any(Char::isWhitespace) || '?' in this || '#' in this) return false
val schemeSeparator = indexOf("://")
if (schemeSeparator <= 0) return false
val scheme = substring(0, schemeSeparator).lowercase()
val authority = substring(schemeSeparator + 3).substringBefore('/')
if (authority.isEmpty() || '@' in authority) return false
val host = when {
authority.startsWith('[') -> {
val end = authority.indexOf(']')
if (end <= 1) return false
val suffix = authority.substring(end + 1)
if (suffix.isNotEmpty() && !suffix.isValidPortSuffix()) return false
authority.substring(1, end)
}
else -> {
if (authority.count { it == ':' } > 1) return false
val portSeparator = authority.indexOf(':')
if (portSeparator >= 0 && !authority.substring(portSeparator).isValidPortSuffix()) return false
authority.substringBefore(':')
}
}.lowercase()
if (host.isEmpty()) return false
if (scheme == "https") return true
return scheme == "http" && host.isLoopbackHost()
}
private fun String.isValidPortSuffix(): Boolean =
startsWith(':') && drop(1).toIntOrNull() in 1..65_535
private fun String.isLoopbackHost(): Boolean {
if (this == "localhost" || this == "::1") return true
val octets = split('.')
return octets.size == 4 &&
octets.first() == "127" &&
octets.all { it.toIntOrNull() in 0..255 }
}
private const val JsonStringPattern =
""""(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*""""
private const val JsonNumberPattern =
"""-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?"""
private val SuccessfulAcknowledgement = Regex(
"""^\s*\{\s*"ok"\s*:\s*true(?:\s*,\s*$JsonStringPattern\s*:\s*(?:$JsonStringPattern|$JsonNumberPattern|true|false|null))*\s*}\s*$""",
)
private val AcknowledgementOk = Regex(""""ok"\s*:""")
private val AcknowledgementId = Regex(""""id"\s*:\s*($JsonStringPattern)""")

View File

@@ -8,6 +8,7 @@ interface PendingCrashStore {
fun write(report: CrashReport) fun write(report: CrashReport)
fun list(): List<CrashReport> fun list(): List<CrashReport>
fun delete(id: String) fun delete(id: String)
fun prune(olderThanTimestampMillis: Long, maxCount: Int)
} }
expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore
@@ -15,14 +16,17 @@ expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore
internal object CrashReportCodec { internal object CrashReportCodec {
private const val FieldSep = "\u001f" private const val FieldSep = "\u001f"
private const val RecordSep = "\u001e" private const val RecordSep = "\u001e"
private const val Version2Prefix = "vnidrop-crash-v2\n"
private const val MaxEncodedChars = 512 * 1024
fun encode(report: CrashReport): String = buildString { fun encode(report: CrashReport): String = buildString {
fun field(key: String, value: String) { fun field(key: String, value: String) {
append(key) append(key)
append('=') append('=')
append(value.replace("\n", "\\n").replace("\r", "\\r")) append(value.hexEncode())
append(FieldSep) append('\n')
} }
append(Version2Prefix)
field("id", report.id) field("id", report.id)
field("ts", report.timestampMillis.toString()) field("ts", report.timestampMillis.toString())
field("install", report.installId) field("install", report.installId)
@@ -31,22 +35,70 @@ internal object CrashReportCodec {
field("type", report.exceptionType) field("type", report.exceptionType)
field("message", report.exceptionMessage) field("message", report.exceptionMessage)
field("stack", report.stackTrace) field("stack", report.stackTrace)
field("diag", if (report.diagnosticsEnabledAtCapture) "1" else "0")
field("schema", report.schemaVersion.toString())
field( field(
"crumbs", "diag",
report.breadcrumbs.joinToString(RecordSep) { crumb -> when (report.diagnosticsEnabledAtCapture) {
listOf( true -> "1"
crumb.timestampMillis.toString(), false -> "0"
crumb.name, null -> "u"
crumb.properties.entries.joinToString(",") { "${it.key}:${it.value}" },
).joinToString("|")
}, },
) )
field("schema", report.schemaVersion.toString())
val breadcrumbs = report.breadcrumbs.take(40)
field("crumb.count", breadcrumbs.size.toString())
breadcrumbs.forEachIndexed { crumbIndex, crumb ->
field("crumb.$crumbIndex.ts", crumb.timestampMillis.toString())
field("crumb.$crumbIndex.name", crumb.name)
val properties = crumb.properties.entries.take(MaxDiagnosticProperties)
field("crumb.$crumbIndex.prop.count", properties.size.toString())
properties.forEachIndexed { propertyIndex, (key, value) ->
field("crumb.$crumbIndex.prop.$propertyIndex.key", key)
field("crumb.$crumbIndex.prop.$propertyIndex.value", value)
}
}
} }
fun decode(raw: String): CrashReport? { fun decode(raw: String): CrashReport? {
if (raw.isBlank()) return null if (raw.isBlank() || raw.length > MaxEncodedChars) return null
return if (raw.startsWith(Version2Prefix)) decodeVersion2(raw) else decodeLegacy(raw)
}
private fun decodeVersion2(raw: String): CrashReport? {
val map = linkedMapOf<String, String>()
for (part in raw.removePrefix(Version2Prefix).lineSequence()) {
if (part.isEmpty()) continue
val eq = part.indexOf('=')
if (eq <= 0) continue
val key = part.substring(0, eq)
val value = part.substring(eq + 1).hexDecode() ?: return null
map[key] = value
}
val crumbCount = map["crumb.count"]?.toIntOrNull()?.takeIf { it in 0..40 } ?: return null
val crumbs = buildList {
repeat(crumbCount) { crumbIndex ->
val timestamp = map["crumb.$crumbIndex.ts"]
?.toLongOrNull()
?.takeIf { it >= 0 }
?: return null
val name = map["crumb.$crumbIndex.name"]?.takeIf { it.isNotBlank() } ?: return null
val propertyCount = map["crumb.$crumbIndex.prop.count"]
?.toIntOrNull()
?.takeIf { it in 0..MaxDiagnosticProperties }
?: return null
val properties = buildMap {
repeat(propertyCount) { propertyIndex ->
val key = map["crumb.$crumbIndex.prop.$propertyIndex.key"] ?: return null
val value = map["crumb.$crumbIndex.prop.$propertyIndex.value"] ?: return null
put(key, value)
}
}
add(Breadcrumb(name = name, timestampMillis = timestamp, properties = properties))
}
}
return reportFromFields(map, crumbs)
}
private fun decodeLegacy(raw: String): CrashReport? {
val map = linkedMapOf<String, String>() val map = linkedMapOf<String, String>()
for (part in raw.split(FieldSep)) { for (part in raw.split(FieldSep)) {
if (part.isEmpty()) continue if (part.isEmpty()) continue
@@ -58,15 +110,14 @@ internal object CrashReportCodec {
.replace("\\r", "\r") .replace("\\r", "\r")
map[key] = value map[key] = value
} }
val id = map["id"] ?: return null
val crumbs = map["crumbs"].orEmpty() val crumbs = map["crumbs"].orEmpty()
.split(RecordSep) .split(RecordSep)
.filter { it.isNotBlank() } .filter { it.isNotBlank() }
.mapNotNull { entry -> .mapNotNull { entry ->
val pieces = entry.split('|', limit = 3) val pieces = entry.split('|', limit = 3)
if (pieces.size < 2) return@mapNotNull null if (pieces.size < 2) return@mapNotNull null
val ts = pieces[0].toLongOrNull() ?: return@mapNotNull null val ts = pieces[0].toLongOrNull()?.takeIf { it >= 0 } ?: return@mapNotNull null
val name = pieces[1] val name = pieces[1].takeIf { it.isNotBlank() } ?: return@mapNotNull null
val props = if (pieces.size > 2 && pieces[2].isNotBlank()) { val props = if (pieces.size > 2 && pieces[2].isNotBlank()) {
pieces[2].split(',').mapNotNull { kv -> pieces[2].split(',').mapNotNull { kv ->
val colon = kv.indexOf(':') val colon = kv.indexOf(':')
@@ -78,18 +129,65 @@ internal object CrashReportCodec {
} }
Breadcrumb(name = name, timestampMillis = ts, properties = props) Breadcrumb(name = name, timestampMillis = ts, properties = props)
} }
return reportFromFields(map, crumbs)
}
private fun reportFromFields(
map: Map<String, String>,
crumbs: List<Breadcrumb>,
): CrashReport? {
val id = map["id"]?.takeIf(::isValidDiagnosticId) ?: return null
val timestamp = map["ts"]?.toLongOrNull()?.takeIf { it >= 0 } ?: return null
val exceptionType = map["type"]?.takeIf { it.isNotBlank() } ?: return null
val schemaVersion = map["schema"]?.toIntOrNull()
?.takeIf { it == DiagnosticsSchemaVersion }
?: return null
val diagnosticsEnabled: Boolean? = when (map["diag"]) {
"1" -> true
"0" -> false
"u" -> null
else -> return null
}
return CrashReport( return CrashReport(
id = id, id = id,
timestampMillis = map["ts"]?.toLongOrNull() ?: 0L, timestampMillis = timestamp,
installId = map["install"].orEmpty(), installId = map["install"].orEmpty(),
appVersion = map["app"].orEmpty(), appVersion = map["app"].orEmpty(),
platform = map["platform"].orEmpty(), platform = map["platform"].orEmpty(),
exceptionType = map["type"].orEmpty(), exceptionType = exceptionType,
exceptionMessage = map["message"].orEmpty(), exceptionMessage = map["message"].orEmpty(),
stackTrace = map["stack"].orEmpty(), stackTrace = map["stack"].orEmpty(),
breadcrumbs = crumbs, breadcrumbs = crumbs,
diagnosticsEnabledAtCapture = map["diag"] == "1", diagnosticsEnabledAtCapture = diagnosticsEnabled,
schemaVersion = map["schema"]?.toIntOrNull() ?: DiagnosticsSchemaVersion, schemaVersion = schemaVersion,
) )
} }
} }
internal fun isValidDiagnosticId(id: String): Boolean =
DiagnosticIdPattern.matches(id)
private val DiagnosticIdPattern =
Regex("^[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$")
private fun String.hexEncode(): String {
val digits = "0123456789abcdef"
return buildString(length * 2) {
for (byte in this@hexEncode.encodeToByteArray()) {
val value = byte.toInt() and 0xff
append(digits[value ushr 4])
append(digits[value and 0x0f])
}
}
}
private fun String.hexDecode(): String? {
if (length % 2 != 0) return null
val bytes = ByteArray(length / 2)
for (index in bytes.indices) {
val high = this[index * 2].digitToIntOrNull(16) ?: return null
val low = this[index * 2 + 1].digitToIntOrNull(16) ?: return null
bytes[index] = ((high shl 4) or low).toByte()
}
return runCatching { bytes.decodeToString(throwOnInvalidSequence = true) }.getOrNull()
}

View File

@@ -0,0 +1,12 @@
package com.vnidrop.app.diagnostics
data class PlatformHttpResponse(
val statusCode: Int,
val body: String,
)
expect suspend fun platformHttpPost(
url: String,
headers: Map<String, String>,
bodyUtf8: String,
): PlatformHttpResponse

View File

@@ -2,17 +2,25 @@ package com.vnidrop.app.diagnostics
import com.vnidrop.app.logging.platformNowMillis import com.vnidrop.app.logging.platformNowMillis
import com.vnidrop.app.preferences.PreferencesRepository import com.vnidrop.app.preferences.PreferencesRepository
import com.vnidrop.app.util.randomUuidString
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.map
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeoutOrNull
import kotlin.concurrent.atomics.AtomicReference
import kotlin.concurrent.atomics.ExperimentalAtomicApi
/** /**
* Product telemetry: sparse events, gated by diagnostics opt-in. * Product telemetry: sparse events, gated by diagnostics opt-in.
* Events are buffered and flushed in batches when transport is available. * Events are buffered and flushed in batches when transport is available.
*/ */
@OptIn(ExperimentalAtomicApi::class)
class TelemetryRecorder( class TelemetryRecorder(
private val preferencesRepository: PreferencesRepository, private val preferencesRepository: PreferencesRepository,
private val transport: DiagnosticsTransport, private val transport: DiagnosticsTransport,
@@ -20,64 +28,186 @@ class TelemetryRecorder(
private val scope: CoroutineScope, private val scope: CoroutineScope,
private val maxBufferSize: Int = DefaultMaxBuffer, private val maxBufferSize: Int = DefaultMaxBuffer,
private val flushThreshold: Int = DefaultFlushThreshold, private val flushThreshold: Int = DefaultFlushThreshold,
private val flushIntervalMillis: Long = DefaultFlushIntervalMillis,
private val retryBackoffMillis: Long = DefaultRetryBackoffMillis,
private val automaticRetryCount: Int = DefaultAutomaticRetryCount,
) { ) {
private val bufferMutex = Mutex() private val bufferMutex = Mutex()
@Volatile private var buffer: List<TelemetryEvent> = emptyList() private val state = AtomicReference(TelemetryState())
@Volatile private var enabled: Boolean = false private val flushSignals = Channel<Unit>(Channel.CONFLATED)
init { init {
require(maxBufferSize > 0) { "maxBufferSize must be positive" }
require(flushThreshold > 0) { "flushThreshold must be positive" }
require(flushIntervalMillis > 0) { "flushIntervalMillis must be positive" }
require(retryBackoffMillis > 0) { "retryBackoffMillis must be positive" }
require(automaticRetryCount >= 0) { "automaticRetryCount must not be negative" }
scope.launch { scope.launch {
preferencesRepository.preferences preferencesRepository.preferences
.map { it.diagnosticsEnabled } .map { it.diagnosticsEnabled }
.distinctUntilChanged() .distinctUntilChanged()
.collect { isEnabled -> .collect { isEnabled ->
enabled = isEnabled updateState { current ->
if (!isEnabled) { if (isEnabled) current.copy(enabled = true) else TelemetryState(enabled = false)
buffer = emptyList()
} }
flushSignals.trySend(Unit)
} }
} }
scope.launch { runAutomaticFlushes() }
} }
fun record(name: String, properties: Map<String, String> = emptyMap()) { fun record(name: String, properties: Map<String, String> = emptyMap()) {
if (!DiagnosticsBuildConfig.INCLUDED) return if (!DiagnosticsBuildConfig.INCLUDED) return
val redacted = LogRedactor.redactMap(properties) val sanitizedName = sanitizeDiagnosticName(name)
breadcrumbs.add(name, redacted) if (sanitizedName.isBlank()) return
if (!enabled) return val sanitizedProperties = sanitizeDiagnosticProperties(properties)
breadcrumbs.add(sanitizedName, sanitizedProperties)
val event = TelemetryEvent( val event = TelemetryEvent(
name = name.take(MaxNameLength), name = sanitizedName,
timestampMillis = platformNowMillis(), timestampMillis = platformNowMillis(),
properties = redacted.mapValues { it.value.take(MaxPropertyValueLength) }, properties = sanitizedProperties,
) )
val next = (buffer + event).takeLast(maxBufferSize) while (true) {
buffer = next val current = state.load()
if (next.size >= flushThreshold) { if (current.enabled == false) return
scope.launch { flush() } val remainingCapacity =
(maxBufferSize - current.retryBatch?.events.orEmpty().size).coerceAtLeast(0)
val nextBuffer = (current.buffer + event).takeLast(remainingCapacity)
if (state.compareAndSet(current, current.copy(buffer = nextBuffer))) {
flushSignals.trySend(Unit)
return
}
} }
} }
suspend fun flush(): Result<Unit> = bufferMutex.withLock { suspend fun flush(): Result<Unit> {
if (!enabled) { return bufferMutex.withLock {
buffer = emptyList() var discardedFailure: Throwable? = null
return Result.success(Unit) var outcome: Result<Unit>? = null
while (outcome == null) {
val current = state.load()
if (current.enabled != true) return@withLock Result.success(Unit)
val pendingRetry = current.retryBatch
val events = if (pendingRetry == null) nextBatchEvents(current.buffer) else emptyList()
if (pendingRetry == null && events.isEmpty()) {
outcome = discardedFailure?.let { Result.failure(it) } ?: Result.success(Unit)
continue
}
val batch: TelemetryBatch
if (pendingRetry != null) {
batch = pendingRetry
} else {
val prepared = TelemetryBatch(id = randomUuidString(), events = events)
val next = current.copy(
buffer = current.buffer.drop(events.size),
retryBatch = prepared,
)
if (!state.compareAndSet(current, next)) continue
batch = prepared
}
if (state.load().retryBatch != batch) continue
val result = try {
transport.sendEvents(batch)
} catch (cancelled: CancellationException) {
throw cancelled
} catch (error: Throwable) {
Result.failure(error)
}
if (result.isSuccess) {
clearRetryBatch(batch)
continue
}
val error = result.exceptionOrNull() ?: IllegalStateException("diagnostics event delivery failed")
if (error.isPermanentDiagnosticsPayloadRejection()) {
clearRetryBatch(batch)
discardedFailure = discardedFailure ?: error
continue
}
outcome = result
}
checkNotNull(outcome)
} }
val batch = buffer
if (batch.isEmpty()) return Result.success(Unit)
buffer = emptyList()
val result = transport.sendEvents(batch)
if (result.isFailure) {
// Re-queue on failure so a future transport can retry once online.
buffer = (batch + buffer).takeLast(maxBufferSize)
}
return result
} }
fun pendingCount(): Int = buffer.size private fun nextBatchEvents(events: List<TelemetryEvent>): List<TelemetryEvent> {
if (events.isEmpty()) return emptyList()
var minimum = 1
var maximum = minOf(events.size, MaxEventsPerBatch)
var accepted = 1
while (minimum <= maximum) {
val candidateSize = minimum + (maximum - minimum) / 2
if (DiagnosticsJson.eventBatchFitsRequest(events.take(candidateSize))) {
accepted = candidateSize
minimum = candidateSize + 1
} else {
maximum = candidateSize - 1
}
}
return events.take(accepted)
}
fun pendingCount(): Int {
val current = state.load()
return current.retryBatch?.events.orEmpty().size + current.buffer.size
}
private suspend fun runAutomaticFlushes() {
while (true) {
flushSignals.receive()
var retries = 0
while (true) {
val current = state.load()
if (current.enabled != true || pendingCount() == 0) break
if (current.retryBatch == null && pendingCount() < flushThreshold) {
val signalled = withTimeoutOrNull(flushIntervalMillis) {
flushSignals.receive()
true
} ?: false
if (signalled) continue
}
val result = flush()
if (result.isSuccess || pendingCount() == 0) {
retries = 0
continue
}
val error = result.exceptionOrNull()
if (error?.isPermanentDiagnosticsPayloadRejection() == true || retries >= automaticRetryCount) {
break
}
retries += 1
delay(retryBackoffMillis)
}
}
}
private fun clearRetryBatch(batch: TelemetryBatch) {
while (true) {
val current = state.load()
if (current.retryBatch != batch) return
if (state.compareAndSet(current, current.copy(retryBatch = null))) return
}
}
private fun updateState(update: (TelemetryState) -> TelemetryState) {
while (true) {
val current = state.load()
if (state.compareAndSet(current, update(current))) return
}
}
companion object { companion object {
const val DefaultMaxBuffer = 100 const val DefaultMaxBuffer = 100
const val DefaultFlushThreshold = 20 const val DefaultFlushThreshold = 20
private const val MaxNameLength = 64 const val MaxEventsPerBatch = 50
private const val MaxPropertyValueLength = 128 const val DefaultFlushIntervalMillis = 30_000L
const val DefaultRetryBackoffMillis = 30_000L
const val DefaultAutomaticRetryCount = 3
} }
} }
private data class TelemetryState(
val enabled: Boolean? = null,
val buffer: List<TelemetryEvent> = emptyList(),
val retryBatch: TelemetryBatch? = null,
)

View File

@@ -1,27 +1,250 @@
package com.vnidrop.app.diagnostics package com.vnidrop.app.diagnostics
import com.vnidrop.app.DeviceInfo import com.vnidrop.app.DeviceInfo
import com.vnidrop.app.preferences.AppPreferences
import com.vnidrop.app.core.ReceiveFolder import com.vnidrop.app.core.ReceiveFolder
import com.vnidrop.app.core.ReceiveFolderKind import com.vnidrop.app.core.ReceiveFolderKind
import com.vnidrop.app.preferences.AppPreferences
import com.vnidrop.app.preferences.PreferencesRepository
import com.vnidrop.app.support.FakePreferencesRepository import com.vnidrop.app.support.FakePreferencesRepository
import com.vnidrop.app.ui.theme.ThemeMode import com.vnidrop.app.ui.theme.ThemeMode
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.TestScope import kotlinx.coroutines.test.TestScope
import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.UnconfinedTestDispatcher
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.advanceUntilIdle import kotlinx.coroutines.test.advanceUntilIdle
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest import kotlinx.coroutines.test.runTest
import kotlin.test.Test import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
import kotlin.test.assertFalse import kotlin.test.assertFalse
import kotlin.test.assertFailsWith
import kotlin.test.assertIs
import kotlin.test.assertNull
import kotlin.test.assertTrue import kotlin.test.assertTrue
@OptIn(ExperimentalCoroutinesApi::class) @OptIn(ExperimentalCoroutinesApi::class)
class DiagnosticsTest { class DiagnosticsTest {
@Test @Test
fun diagnosticsBuildConfigDefaultsToIncluded() { fun diagnosticsBuildConfigDefaultsToIncludedWithEmptyEndpoint() {
// Production default is true; builds can override with -Pvnidrop.diagnostics.included=false. // Production default is true; builds can override with -Pvnidrop.diagnostics.included=false.
assertTrue(DiagnosticsBuildConfig.INCLUDED) assertTrue(DiagnosticsBuildConfig.INCLUDED)
// Empty endpoint keeps shipping safe (NoOp transport) until Cloudflare is configured.
assertEquals("", DiagnosticsBuildConfig.ENDPOINT)
}
@Test
fun diagnosticsJsonEscapesAndShapesPayloads() {
val eventsJson = DiagnosticsJson.eventsBody(
batchId = "batch-1",
installId = "inst-1",
appVersion = "1.0",
platform = "Test",
events = listOf(
TelemetryEvent("app_open", 10L, mapOf("a" to "quote\"here")),
),
)
assertTrue(eventsJson.contains("\"batchId\":\"batch-1\""))
assertTrue(eventsJson.contains("\"installId\":\"inst-1\""))
assertTrue(eventsJson.contains("\"name\":\"app_open\""))
assertTrue(eventsJson.contains("quote\\\"here"))
val crashJson = DiagnosticsJson.crashBody(
CrashReport(
id = "c1",
timestampMillis = 1L,
installId = "inst",
appVersion = "1.0",
platform = "Test",
exceptionType = "E",
exceptionMessage = "line\nbreak",
stackTrace = "stack",
breadcrumbs = emptyList(),
diagnosticsEnabledAtCapture = true,
),
)
assertTrue(crashJson.contains("line\\nbreak"))
assertTrue(crashJson.contains("\"diagnosticsEnabledAtCapture\":true"))
}
@Test
fun diagnosticsJsonKeepsEscapedBugPayloadWithinWorkerLimit() {
val report = BugReport(
id = "b",
timestampMillis = 1L,
installId = "i",
appVersion = "1.0",
platform = "Test",
whatHappened = "w",
expected = "e",
steps = "",
contact = "",
includeLogs = true,
logs = "\n".repeat(BugReportService.MaxLogBytes),
device = DeviceSnapshot(null, null, "OS", null, null),
breadcrumbs = emptyList(),
)
val body = DiagnosticsJson.bugBody(report)
assertTrue(body.encodeToByteArray().size <= DiagnosticsJson.MaxRequestBytes)
assertTrue(body.contains("\"includeLogs\":true"))
assertTrue(body.endsWith("}"))
}
@Test
fun httpTransportPostsExpectedPaths() = runTest {
val calls = mutableListOf<Pair<String, String>>()
val acknowledgementIds = ArrayDeque(listOf("batch-1", "c", "b"))
val transport = HttpDiagnosticsTransport(
baseUrl = "https://diag.example",
ingestKey = "secret",
appVersion = "1.0",
platform = "Test",
installIdProvider = { "install-x" },
post = { url, headers, body ->
assertEquals("secret", headers["X-VniDrop-Key"])
calls += url to body
PlatformHttpResponse(
202,
"""{"ok":true,"id":"${acknowledgementIds.removeFirst()}","stored":1}""",
)
},
)
val eventResult = transport.sendEvents(
TelemetryBatch("batch-1", listOf(TelemetryEvent("nav", 1L))),
)
assertTrue(eventResult.isSuccess)
assertEquals("https://diag.example/v1/events", calls[0].first)
assertTrue(calls[0].second.contains("install-x"))
val crashResult = transport.sendCrash(
CrashReport(
id = "c",
timestampMillis = 1L,
installId = "i",
appVersion = "1.0",
platform = "Test",
exceptionType = "E",
exceptionMessage = "m",
stackTrace = "s",
breadcrumbs = emptyList(),
diagnosticsEnabledAtCapture = true,
),
)
assertTrue(crashResult.isSuccess)
assertEquals("https://diag.example/v1/crashes", calls[1].first)
val bugResult = transport.sendBugReport(
BugReport(
id = "b",
timestampMillis = 1L,
installId = "i",
appVersion = "1.0",
platform = "Test",
whatHappened = "w",
expected = "e",
steps = "",
contact = "",
includeLogs = false,
logs = "",
device = DeviceSnapshot(null, null, "OS", null, null),
breadcrumbs = emptyList(),
),
)
assertTrue(bugResult.isSuccess)
assertEquals("https://diag.example/v1/bugs", calls[2].first)
}
@Test
fun httpTransportFailsOnHttpError() = runTest {
val transport = HttpDiagnosticsTransport(
baseUrl = "https://diag.example",
ingestKey = "secret",
post = { _, _, _ -> PlatformHttpResponse(401, """{"error":"unauthorized"}""") },
)
assertTrue(
transport.sendEvents(TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1L)))).isFailure,
)
}
@Test
fun httpTransportPropagatesCancellation() = runTest {
val transport = HttpDiagnosticsTransport(
baseUrl = "https://diag.example",
ingestKey = "secret",
post = { _, _, _ -> throw CancellationException("cancelled") },
)
assertFailsWith<CancellationException> {
transport.sendEvents(TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1))))
}
}
@Test
fun httpTransportRejectsMissingOrNegativeAcknowledgement() = runTest {
val responses = ArrayDeque(
listOf(
PlatformHttpResponse(202, ""),
PlatformHttpResponse(202, """{"ok":false}"""),
PlatformHttpResponse(202, """{,"ok":true}"""),
PlatformHttpResponse(202, """{"ok":true,"id":"different"}"""),
PlatformHttpResponse(202, """{"ok":true,"id":"batch-4","id":"batch-4"}"""),
),
)
val transport = HttpDiagnosticsTransport(
baseUrl = "https://diag.example",
ingestKey = "secret",
post = { _, _, _ -> responses.removeFirst() },
)
repeat(5) { index ->
val result = transport.sendEvents(
TelemetryBatch("batch-$index", listOf(TelemetryEvent("x", 1L))),
)
assertIs<DiagnosticsProtocolException>(result.exceptionOrNull())
}
}
@Test
fun httpTransportRequiresHttpsExceptForLoopbackDevelopment() {
assertFailsWith<IllegalArgumentException> {
HttpDiagnosticsTransport("http://diag.example", "secret")
}
assertFailsWith<IllegalArgumentException> {
HttpDiagnosticsTransport("http://[::1].example", "secret")
}
HttpDiagnosticsTransport("http://localhost:8787", "secret")
HttpDiagnosticsTransport("http://127.0.0.1:8787", "secret")
HttpDiagnosticsTransport("http://[::1]:8787", "secret")
assertFailsWith<IllegalArgumentException> {
HttpDiagnosticsTransport("https://diag.example", "")
}
}
@Test
fun createTransportIsNoOpWhenEndpointBlank() {
// With default generated config endpoint is empty.
val transport = createDiagnosticsTransport(
appVersion = "1.0",
platform = "Test",
installIdProvider = { "id" },
)
assertIs<NoOpDiagnosticsTransport>(transport)
}
@Test
fun noOpTransportReportsUnavailableDelivery() = runTest {
val result = NoOpDiagnosticsTransport().sendEvents(
TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1L))),
)
assertIs<DiagnosticsUnavailableException>(result.exceptionOrNull())
} }
@Test @Test
@@ -72,6 +295,35 @@ class DiagnosticsTest {
assertEquals(1, breadcrumbs.snapshot().size) assertEquals(1, breadcrumbs.snapshot().size)
} }
@Test
fun telemetryRetainsColdStartEventsUntilConsentLoads() = runTest {
val backing = fakePrefs(diagnosticsEnabled = true)
val preferenceGate = CompletableDeferred<Unit>()
val delayedPreferences = object : PreferencesRepository by backing {
override val preferences = flow {
preferenceGate.await()
emitAll(backing.preferences)
}
}
val transport = RecordingDiagnosticsTransport()
val recorder = TelemetryRecorder(
preferencesRepository = delayedPreferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 20,
)
runCurrent()
recorder.record("app_open")
assertEquals(1, recorder.pendingCount())
preferenceGate.complete(Unit)
runCurrent()
assertTrue(recorder.flush().isSuccess)
assertEquals(listOf("app_open"), transport.events.single().map { it.name })
}
@Test @Test
fun telemetryBuffersAndFlushesWhenEnabled() = runTest { fun telemetryBuffersAndFlushesWhenEnabled() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true) val preferences = fakePrefs(diagnosticsEnabled = true)
@@ -92,6 +344,222 @@ class DiagnosticsTest {
assertEquals(listOf("one", "two"), transport.events.single().map { it.name }) assertEquals(listOf("one", "two"), transport.events.single().map { it.name })
} }
@Test
fun telemetryFlushesSparseEventsAfterTheInterval() = runTest {
val transport = RecordingDiagnosticsTransport()
val recorder = TelemetryRecorder(
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 20,
flushIntervalMillis = 1_000,
)
advanceUntilIdle()
recorder.record("sparse")
advanceTimeBy(999)
runCurrent()
assertTrue(transport.eventBatches.isEmpty())
advanceTimeBy(1)
runCurrent()
assertEquals(listOf("sparse"), transport.events.single().map { it.name })
}
@Test
fun telemetryCoalescesAutomaticRetriesWithBackoff() = runTest {
val transport = RecordingDiagnosticsTransport().apply {
eventsResult = Result.failure(IllegalStateException("offline"))
}
val recorder = TelemetryRecorder(
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 1,
flushIntervalMillis = 10_000,
retryBackoffMillis = 1_000,
automaticRetryCount = 1,
)
advanceUntilIdle()
recorder.record("retry")
runCurrent()
assertEquals(1, transport.eventBatches.size)
advanceTimeBy(999)
runCurrent()
assertEquals(1, transport.eventBatches.size)
advanceTimeBy(1)
runCurrent()
assertEquals(2, transport.eventBatches.size)
advanceTimeBy(10_000)
runCurrent()
assertEquals(2, transport.eventBatches.size)
}
@Test
fun telemetryFlushesAtMostFiftyEventsPerBatch() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val transport = RecordingDiagnosticsTransport()
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 101,
maxBufferSize = 100,
)
advanceUntilIdle()
repeat(75) { recorder.record("event-$it") }
assertTrue(recorder.flush().isSuccess)
assertEquals(listOf(50, 25), transport.eventBatches.map { it.events.size })
assertTrue(transport.eventBatches.all { it.events.size <= TelemetryRecorder.MaxEventsPerBatch })
}
@Test
fun telemetrySplitsBatchesByEscapedRequestBytes() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val requestBodies = mutableListOf<String>()
val transport = HttpDiagnosticsTransport(
baseUrl = "https://diag.example",
ingestKey = "secret",
appVersion = "1.0",
platform = "Test",
installIdProvider = { "test-install" },
post = { _, _, body ->
requestBodies += body
val id = Regex(""""batchId":"([^"]+)"""").find(body)?.groupValues?.get(1)
PlatformHttpResponse(202, """{"ok":true,"id":"$id"}""")
},
)
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 101,
maxBufferSize = 100,
)
advanceUntilIdle()
val properties = LinkedHashMap<String, String>().apply {
repeat(MaxDiagnosticProperties) { index ->
put("key-$index-${"\u0001".repeat(40)}", "\u0001".repeat(MaxDiagnosticPropertyValueBytes))
}
}
repeat(50) { index ->
recorder.record("event-$index-${"\u0001".repeat(64)}", properties)
}
assertTrue(recorder.flush().isSuccess)
assertTrue(requestBodies.size > 1)
assertTrue(requestBodies.all { it.encodeToByteArray().size <= DiagnosticsJson.MaxRequestBytes })
assertEquals(50, requestBodies.sumOf { body -> "\"schemaVersion\"".toRegex().findAll(body).count() })
}
@Test
fun telemetrySanitizesNamesAndPropertiesToServerByteLimits() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val transport = RecordingDiagnosticsTransport()
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 100,
)
advanceUntilIdle()
val properties = LinkedHashMap<String, String>().apply {
repeat(20) { index -> put("key-$index-${"🙂".repeat(20)}", "🙂".repeat(100)) }
}
recorder.record("🙂".repeat(100), properties)
assertTrue(recorder.flush().isSuccess)
val event = transport.eventBatches.single().events.single()
assertTrue(event.name.encodeToByteArray().size <= MaxDiagnosticNameBytes)
assertEquals(MaxDiagnosticProperties, event.properties.size)
assertTrue(event.properties.keys.all { it.encodeToByteArray().size <= MaxDiagnosticPropertyKeyBytes })
assertTrue(event.properties.values.all { it.encodeToByteArray().size <= MaxDiagnosticPropertyValueBytes })
}
@Test
fun telemetryKeepsConcurrentRecordsWithoutExceedingItsBuffer() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val transport = RecordingDiagnosticsTransport()
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 101,
maxBufferSize = 100,
)
advanceUntilIdle()
coroutineScope {
repeat(100) { index ->
launch(Dispatchers.Default) { recorder.record("event-$index") }
}
}
assertEquals(
100,
recorder.pendingCount() + transport.eventBatches.sumOf { it.events.size },
)
assertTrue(recorder.flush().isSuccess)
assertEquals(100, transport.eventBatches.sumOf { it.events.size })
}
@Test
fun telemetryRetryReusesBatchIdAndEvents() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val transport = RecordingDiagnosticsTransport().apply {
eventsResult = Result.failure(IllegalStateException("offline"))
}
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 10,
)
advanceUntilIdle()
recorder.record("one")
recorder.record("two")
assertTrue(recorder.flush().isFailure)
val firstAttempt = transport.eventBatches.single()
transport.eventsResult = Result.success(Unit)
assertTrue(recorder.flush().isSuccess)
assertEquals(listOf(firstAttempt, firstAttempt), transport.eventBatches)
assertEquals(0, recorder.pendingCount())
}
@Test
fun telemetryDoesNotRequeuePermanentlyRejectedPayload() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val transport = RecordingDiagnosticsTransport().apply {
eventsResult = Result.failure(DiagnosticsHttpException(400, "/v1/events"))
}
val recorder = TelemetryRecorder(
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
flushThreshold = 10,
)
advanceUntilIdle()
recorder.record("invalid")
assertTrue(recorder.flush().isFailure)
assertEquals(0, recorder.pendingCount())
transport.eventsResult = Result.success(Unit)
assertTrue(recorder.flush().isSuccess)
assertEquals(1, transport.eventBatches.size)
}
@Test @Test
fun telemetryClearsBufferWhenOptedOut() = runTest { fun telemetryClearsBufferWhenOptedOut() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true) val preferences = fakePrefs(diagnosticsEnabled = true)
@@ -114,21 +582,49 @@ class DiagnosticsTest {
@Test @Test
fun crashCodecRoundTrips() { fun crashCodecRoundTrips() {
val original = CrashReport( val original = CrashReport(
id = "crash-1", id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
timestampMillis = 42L, timestampMillis = 42L,
installId = "install", installId = "install",
appVersion = "1.0", appVersion = "1.0",
platform = "Test", platform = "Test",
exceptionType = "IllegalStateException", exceptionType = "IllegalStateException",
exceptionMessage = "boom\nline", exceptionMessage = "boom\nline\\nliteral\u001fseparator",
stackTrace = "stack\ntrace", stackTrace = "stack\ntrace\u001erecord",
breadcrumbs = listOf( breadcrumbs = listOf(
Breadcrumb("open", 1L, mapOf("screen" to "send")), Breadcrumb("open|send", 1L, mapOf("screen:key" to "send,value|next")),
), ),
diagnosticsEnabledAtCapture = true, diagnosticsEnabledAtCapture = true,
) )
val decoded = CrashReportCodec.decode(CrashReportCodec.encode(original)) val decoded = CrashReportCodec.decode(CrashReportCodec.encode(original))
assertEquals(original, decoded) assertEquals(original, decoded)
assertNull(CrashReportCodec.decode(CrashReportCodec.encode(original.copy(id = "../../escape"))))
}
@Test
fun crashCodecMigratesLegacyV1Envelope() {
val raw = listOf(
"id=bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
"ts=42",
"install=legacy-install",
"app=0.9",
"platform=Desktop",
"type=IllegalStateException",
"message=first\\nsecond",
"stack=frame one\\nframe two",
"diag=1",
"schema=1",
"crumbs=1|opened|screen:send",
).joinToString("\u001f")
val report = requireNotNull(CrashReportCodec.decode(raw))
assertEquals("first\nsecond", report.exceptionMessage)
assertEquals("frame one\nframe two", report.stackTrace)
assertEquals(true, report.diagnosticsEnabledAtCapture)
assertEquals(
listOf(Breadcrumb("opened", 1, mapOf("screen" to "send"))),
report.breadcrumbs,
)
} }
@Test @Test
@@ -150,13 +646,165 @@ class DiagnosticsTest {
val optedIn = reporter.capture(RuntimeException("a"), diagnosticsEnabledOverride = true) val optedIn = reporter.capture(RuntimeException("a"), diagnosticsEnabledOverride = true)
reporter.capture(RuntimeException("b"), diagnosticsEnabledOverride = false) reporter.capture(RuntimeException("b"), diagnosticsEnabledOverride = false)
assertEquals(2, store.list().size) assertEquals(1, store.list().size)
reporter.flushPending() reporter.flushPending()
assertEquals(1, transport.crashes.size) assertEquals(1, transport.crashes.size)
assertEquals(optedIn.id, transport.crashes.single().id) assertEquals(optedIn.id, transport.crashes.single().id)
assertTrue(store.list().isEmpty())
}
@Test
fun crashReporterRetainsCrashWhenDeliveryIsUnavailable() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val store = InMemoryPendingCrashStore()
val reporter = CrashReporter(
store = store,
preferencesRepository = preferences,
transport = NoOpDiagnosticsTransport(),
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
reporter.capture(RuntimeException("boom"), diagnosticsEnabledOverride = true)
reporter.flushPending()
assertEquals(1, store.list().size) assertEquals(1, store.list().size)
assertFalse(store.list().single().diagnosticsEnabledAtCapture) }
@Test
fun crashReporterDropsPermanentPayloadFailuresAndStopsAfterTransientFailures() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val permanentTransport = RecordingDiagnosticsTransport().apply {
crashResult = Result.failure(DiagnosticsHttpException(400, "/v1/crashes"))
}
val permanentStore = InMemoryPendingCrashStore()
val permanentReporter = CrashReporter(
store = permanentStore,
preferencesRepository = preferences,
transport = permanentTransport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
permanentReporter.capture(RuntimeException("invalid"), diagnosticsEnabledOverride = true)
permanentReporter.flushPending()
assertTrue(permanentStore.list().isEmpty())
val transientTransport = RecordingDiagnosticsTransport().apply {
crashResult = Result.failure(IllegalStateException("offline"))
}
val transientStore = InMemoryPendingCrashStore()
val transientReporter = CrashReporter(
store = transientStore,
preferencesRepository = preferences,
transport = transientTransport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
repeat(2) {
transientReporter.capture(RuntimeException("offline-$it"), diagnosticsEnabledOverride = true)
}
transientReporter.flushPending()
assertEquals(1, transientTransport.crashes.size)
assertEquals(2, transientStore.list().size)
}
@Test
fun crashReporterResolvesStartupConsentBeforeUploading() = runTest {
val transport = RecordingDiagnosticsTransport()
val store = InMemoryPendingCrashStore()
val reporter = CrashReporter(
store = store,
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
val startupCrash = reporter.capture(RuntimeException("startup"))
assertNull(startupCrash.diagnosticsEnabledAtCapture)
reporter.flushPending()
assertEquals(true, transport.crashes.single().diagnosticsEnabledAtCapture)
assertEquals("test-install", transport.crashes.single().installId)
assertTrue(store.list().isEmpty())
}
@Test
fun crashReporterDoesNotRetroactivelyUploadAnOptedOutStartupCrash() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = false)
val transport = RecordingDiagnosticsTransport()
val store = InMemoryPendingCrashStore()
val reporter = CrashReporter(
store = store,
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
reporter.capture(RuntimeException("startup"))
reporter.flushPending()
assertTrue(store.list().isEmpty())
preferences.setDiagnosticsEnabled(true)
reporter.flushPending()
assertTrue(transport.crashes.isEmpty())
assertTrue(store.list().isEmpty())
}
@Test
fun crashReporterStopsAFlushWhenTheUserOptsOut() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = true)
val firstSendStarted = CompletableDeferred<Unit>()
val releaseFirstSend = CompletableDeferred<Unit>()
val sentIds = mutableListOf<String>()
val transport = object : DiagnosticsTransport {
override suspend fun sendEvents(batch: TelemetryBatch) = Result.success(Unit)
override suspend fun sendBugReport(report: BugReport) = Result.success(Unit)
override suspend fun sendCrash(report: CrashReport): Result<Unit> {
sentIds += report.id
if (sentIds.size == 1) {
firstSendStarted.complete(Unit)
releaseFirstSend.await()
}
return Result.success(Unit)
}
}
val store = InMemoryPendingCrashStore()
val reporter = CrashReporter(
store = store,
preferencesRepository = preferences,
transport = transport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
)
reporter.startObservingPreferences()
advanceUntilIdle()
repeat(2) {
reporter.capture(RuntimeException("crash-$it"), diagnosticsEnabledOverride = true)
}
val flush = launch { reporter.flushPending() }
firstSendStarted.await()
preferences.setDiagnosticsEnabled(false)
runCurrent()
releaseFirstSend.complete(Unit)
flush.join()
assertEquals(1, sentIds.size)
assertTrue(store.list().isEmpty())
} }
@Test @Test
@@ -174,6 +822,29 @@ class DiagnosticsTest {
assertTrue(service.submit(BugReportDraft("what", ""), device()).isFailure) assertTrue(service.submit(BugReportDraft("what", ""), device()).isFailure)
} }
@Test
fun bugReportConvertsTransportExceptionsToFailure() = runTest {
val throwingTransport = object : DiagnosticsTransport {
override suspend fun sendEvents(batch: TelemetryBatch) = Result.success(Unit)
override suspend fun sendCrash(report: CrashReport) = Result.success(Unit)
override suspend fun sendBugReport(report: BugReport): Result<Unit> {
throw IllegalStateException("offline")
}
}
val service = BugReportService(
preferencesRepository = fakePrefs(),
transport = throwingTransport,
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
)
val result = service.submit(BugReportDraft("what", "expected"), device())
assertTrue(result.isFailure)
assertEquals("offline", result.exceptionOrNull()?.message)
}
@Test @Test
fun bugReportSubmitsWithRedactedLogsRegardlessOfDiagnostics() = runTest { fun bugReportSubmitsWithRedactedLogsRegardlessOfDiagnostics() = runTest {
val preferences = fakePrefs(diagnosticsEnabled = false) val preferences = fakePrefs(diagnosticsEnabled = false)
@@ -205,6 +876,43 @@ class DiagnosticsTest {
assertEquals("test-install", report.installId) assertEquals("test-install", report.installId)
} }
@Test
fun bugReportRedactsLogsBeforeApplyingUtf8Limit() {
val secret = "abcdefghijklmnopqrstuvwxyz012345"
val rawLogs = "ticket=$secret\n".repeat(6_000) + "tail-marker"
val service = BugReportService(
preferencesRepository = fakePrefs(),
transport = RecordingDiagnosticsTransport(),
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
logReader = { rawLogs },
)
val logs = service.assemble(BugReportDraft("what", "expected"), device(), "install").logs
assertFalse(logs.contains(secret))
assertTrue(logs.endsWith("tail-marker"))
assertTrue(logs.encodeToByteArray().size <= BugReportService.MaxLogBytes)
}
@Test
fun bugReportLogLimitCountsUtf8Bytes() {
val service = BugReportService(
preferencesRepository = fakePrefs(),
transport = RecordingDiagnosticsTransport(),
breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0",
platform = "Test",
logReader = { "🙂".repeat(60_000) },
)
val logs = service.assemble(BugReportDraft("what", "expected"), device(), "install").logs
assertEquals(BugReportService.MaxLogBytes, logs.encodeToByteArray().size)
assertEquals(BugReportService.MaxLogBytes, service.previewLogBytes())
}
private fun fakePrefs(diagnosticsEnabled: Boolean = false) = FakePreferencesRepository( private fun fakePrefs(diagnosticsEnabled: Boolean = false) = FakePreferencesRepository(
AppPreferences( AppPreferences(
username = "User", username = "User",
@@ -228,4 +936,15 @@ private class InMemoryPendingCrashStore : PendingCrashStore {
override fun delete(id: String) { override fun delete(id: String) {
items.remove(id) items.remove(id)
} }
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
items.values
.sortedByDescending { it.timestampMillis }
.drop(maxCount)
.map(CrashReport::id)
.forEach(items::remove)
items.values
.filter { it.timestampMillis < olderThanTimestampMillis }
.map(CrashReport::id)
.forEach(items::remove)
}
} }

View File

@@ -10,13 +10,16 @@ import com.vnidrop.app.core.ShareAccessPolicy
import com.vnidrop.app.core.Transfer import com.vnidrop.app.core.Transfer
import com.vnidrop.app.core.TransferDirection import com.vnidrop.app.core.TransferDirection
import com.vnidrop.app.core.TransferStatus import com.vnidrop.app.core.TransferStatus
import com.vnidrop.app.diagnostics.BreadcrumbBuffer
import com.vnidrop.app.diagnostics.BugReportService
import com.vnidrop.app.diagnostics.DiagnosticsTransport
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport
import com.vnidrop.app.diagnostics.RecordingDiagnosticsTransport
import com.vnidrop.app.feature.app.AppViewModel import com.vnidrop.app.feature.app.AppViewModel
import com.vnidrop.app.feature.receive.ReceiveHistoryDeleteTarget import com.vnidrop.app.feature.receive.ReceiveHistoryDeleteTarget
import com.vnidrop.app.feature.receive.ReceiveViewModel import com.vnidrop.app.feature.receive.ReceiveViewModel
import com.vnidrop.app.feature.send.SendViewModel import com.vnidrop.app.feature.send.SendViewModel
import com.vnidrop.app.diagnostics.BugReportService import com.vnidrop.app.feature.settings.SettingsSection
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport
import com.vnidrop.app.diagnostics.BreadcrumbBuffer
import com.vnidrop.app.feature.settings.SettingsViewModel import com.vnidrop.app.feature.settings.SettingsViewModel
import com.vnidrop.app.notifications.NotificationPermission import com.vnidrop.app.notifications.NotificationPermission
import com.vnidrop.app.preferences.AppPreferences import com.vnidrop.app.preferences.AppPreferences
@@ -165,6 +168,24 @@ class ViewModelsTest {
assertEquals("", viewModel.state.value.bugExpected) assertEquals("", viewModel.state.value.bugExpected)
} }
@Test
fun settingsKeepsBugReportWhenDeliveryIsUnavailable() = runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val viewModel = settingsViewModel(transport = NoOpDiagnosticsTransport())
advanceUntilIdle()
viewModel.selectSection(SettingsSection.BugReport)
viewModel.setBugWhatHappened("Transfer stuck")
viewModel.setBugExpected("It should finish")
viewModel.submitBugReport()
advanceUntilIdle()
assertEquals("Transfer stuck", viewModel.state.value.bugWhatHappened)
assertEquals("It should finish", viewModel.state.value.bugExpected)
assertEquals(SettingsSection.BugReport, viewModel.state.value.selectedSection)
assertFalse(viewModel.state.value.isSubmittingBugReport)
}
@Test @Test
fun sendViewModelOwnsSelectedFileState() = runTest { fun sendViewModelOwnsSelectedFileState() = runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler)) Dispatchers.setMain(StandardTestDispatcher(testScheduler))
@@ -512,6 +533,7 @@ class ViewModelsTest {
private fun settingsViewModel( private fun settingsViewModel(
preferences: PreferencesRepository = preferences(), preferences: PreferencesRepository = preferences(),
notifications: FakeNotificationService = FakeNotificationService(), notifications: FakeNotificationService = FakeNotificationService(),
transport: DiagnosticsTransport = RecordingDiagnosticsTransport(),
) = SettingsViewModel( ) = SettingsViewModel(
environment(), environment(),
{ DeviceInfo("Device", "Model", "OS", "Wi-Fi", "80%") }, { DeviceInfo("Device", "Model", "OS", "Wi-Fi", "80%") },
@@ -521,7 +543,7 @@ class ViewModelsTest {
UiMessageController(), UiMessageController(),
BugReportService( BugReportService(
preferencesRepository = preferences, preferencesRepository = preferences,
transport = NoOpDiagnosticsTransport(), transport = transport,
breadcrumbs = BreadcrumbBuffer(), breadcrumbs = BreadcrumbBuffer(),
appVersion = "1.0", appVersion = "1.0",
platform = "Test", platform = "Test",

View File

@@ -1,15 +1,13 @@
package com.vnidrop.app.diagnostics package com.vnidrop.app.diagnostics
import kotlinx.cinterop.BetaInteropApi
import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.cinterop.ExperimentalForeignApi
import kotlinx.cinterop.addressOf import kotlinx.cinterop.addressOf
import kotlinx.cinterop.convert import kotlinx.cinterop.convert
import kotlinx.cinterop.usePinned import kotlinx.cinterop.usePinned
import platform.Foundation.NSData import platform.Foundation.NSData
import platform.Foundation.NSFileManager import platform.Foundation.NSFileManager
import platform.Foundation.NSString
import platform.Foundation.NSUTF8StringEncoding
import platform.Foundation.create import platform.Foundation.create
import platform.Foundation.dataUsingEncoding
import platform.Foundation.dataWithContentsOfFile import platform.Foundation.dataWithContentsOfFile
import platform.Foundation.writeToFile import platform.Foundation.writeToFile
import platform.posix.memcpy import platform.posix.memcpy
@@ -25,10 +23,11 @@ private class IosPendingCrashStore(
private val directory = appDataDir.trimEnd('/') + "/diagnostics/crashes" private val directory = appDataDir.trimEnd('/') + "/diagnostics/crashes"
override fun write(report: CrashReport) { override fun write(report: CrashReport) {
if (!isValidDiagnosticId(report.id)) return
ensureDirectory() ensureDirectory()
val path = "$directory/${report.id}.crash" val path = "$directory/${report.id}.crash"
val payload = CrashReportCodec.encode(report) val payload = CrashReportCodec.encode(report)
val data = (payload as NSString).dataUsingEncoding(NSUTF8StringEncoding) ?: return val data = payload.encodeToByteArray().toNSData()
data.writeToFile(path, atomically = true) data.writeToFile(path, atomically = true)
} }
@@ -46,14 +45,47 @@ private class IosPendingCrashStore(
} }
override fun delete(id: String) { override fun delete(id: String) {
if (!isValidDiagnosticId(id)) return
fileManager.removeItemAtPath("$directory/$id.crash", null) fileManager.removeItemAtPath("$directory/$id.crash", null)
} }
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
require(maxCount > 0) { "maxCount must be positive" }
ensureDirectory()
val reports = fileManager.contentsOfDirectoryAtPath(directory, null).orEmpty()
.filterIsInstance<String>()
.filter { it.endsWith(".crash") }
.mapNotNull { name ->
val path = "$directory/$name"
val report = NSData.dataWithContentsOfFile(path)
?.toUtf8String()
?.let(CrashReportCodec::decode)
if (report == null) {
fileManager.removeItemAtPath(path, null)
null
} else {
name to report
}
}
.sortedByDescending { (_, report) -> report.timestampMillis }
reports.forEachIndexed { index, (name, report) ->
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) {
fileManager.removeItemAtPath("$directory/$name", null)
}
}
}
private fun ensureDirectory() { private fun ensureDirectory() {
fileManager.createDirectoryAtPath(directory, withIntermediateDirectories = true, attributes = null, error = null) fileManager.createDirectoryAtPath(directory, withIntermediateDirectories = true, attributes = null, error = null)
} }
} }
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
private fun ByteArray.toNSData(): NSData =
usePinned { pinned ->
NSData.create(bytes = pinned.addressOf(0), length = size.toULong())
}
@OptIn(ExperimentalForeignApi::class) @OptIn(ExperimentalForeignApi::class)
private fun NSData.toUtf8String(): String { private fun NSData.toUtf8String(): String {
val size = length.toInt() val size = length.toInt()

View File

@@ -0,0 +1,73 @@
package com.vnidrop.app.diagnostics
import kotlinx.cinterop.BetaInteropApi
import kotlinx.cinterop.ExperimentalForeignApi
import kotlinx.cinterop.addressOf
import kotlinx.cinterop.convert
import kotlinx.cinterop.usePinned
import kotlinx.coroutines.suspendCancellableCoroutine
import platform.Foundation.NSData
import platform.Foundation.NSHTTPURLResponse
import platform.Foundation.NSMutableURLRequest
import platform.Foundation.NSURL
import platform.Foundation.NSURLSession
import platform.Foundation.create
import platform.Foundation.dataTaskWithRequest
import platform.Foundation.setHTTPBody
import platform.Foundation.setHTTPMethod
import platform.Foundation.setValue
import platform.posix.memcpy
import kotlin.coroutines.resume
@OptIn(ExperimentalForeignApi::class)
actual suspend fun platformHttpPost(
url: String,
headers: Map<String, String>,
bodyUtf8: String,
): PlatformHttpResponse = suspendCancellableCoroutine { cont ->
val nsUrl = NSURL.URLWithString(url)
if (nsUrl == null) {
cont.resume(PlatformHttpResponse(statusCode = 0, body = "invalid_url"))
return@suspendCancellableCoroutine
}
val request = NSMutableURLRequest.requestWithURL(nsUrl).apply {
setHTTPMethod("POST")
setValue("application/json; charset=utf-8", forHTTPHeaderField = "Content-Type")
headers.forEach { (key, value) ->
setValue(value, forHTTPHeaderField = key)
}
setHTTPBody(bodyUtf8.encodeToByteArray().toNSData())
}
val task = NSURLSession.sharedSession.dataTaskWithRequest(request) { data, response, error ->
if (!cont.isActive) return@dataTaskWithRequest
if (error != null) {
val message = error.localizedDescription
cont.resume(PlatformHttpResponse(statusCode = 0, body = message))
return@dataTaskWithRequest
}
val http = response as? NSHTTPURLResponse
val status = http?.statusCode?.toInt() ?: 0
val body = data?.toUtf8String().orEmpty()
cont.resume(PlatformHttpResponse(statusCode = status, body = body))
}
cont.invokeOnCancellation { task.cancel() }
task.resume()
}
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
private fun ByteArray.toNSData(): NSData =
usePinned { pinned ->
NSData.create(bytes = pinned.addressOf(0), length = size.toULong())
}
@OptIn(ExperimentalForeignApi::class)
private fun NSData.toUtf8String(): String {
val size = length.toInt()
if (size == 0) return ""
val result = ByteArray(size)
val source = bytes ?: return ""
result.usePinned { pinned ->
memcpy(pinned.addressOf(0), source, size.convert())
}
return result.decodeToString()
}

View File

@@ -13,8 +13,16 @@ private class JvmPendingCrashStore(
@Synchronized @Synchronized
override fun write(report: CrashReport) { override fun write(report: CrashReport) {
if (!isValidDiagnosticId(report.id)) return
directory.mkdirs() directory.mkdirs()
File(directory, "${report.id}.crash").writeText(CrashReportCodec.encode(report), StandardCharsets.UTF_8) val target = File(directory, "${report.id}.crash")
val temporary = File(directory, ".${report.id}.tmp")
val payload = CrashReportCodec.encode(report)
temporary.writeText(payload, StandardCharsets.UTF_8)
if (!temporary.renameTo(target)) {
target.writeText(payload, StandardCharsets.UTF_8)
temporary.delete()
}
} }
@Synchronized @Synchronized
@@ -31,6 +39,34 @@ private class JvmPendingCrashStore(
@Synchronized @Synchronized
override fun delete(id: String) { override fun delete(id: String) {
if (!isValidDiagnosticId(id)) return
File(directory, "$id.crash").delete() File(directory, "$id.crash").delete()
} }
@Synchronized
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
require(maxCount > 0) { "maxCount must be positive" }
if (!directory.isDirectory) return
directory.listFiles { file -> file.isFile && file.name.endsWith(".tmp") }
.orEmpty()
.forEach(File::delete)
val reports = directory
.listFiles { file -> file.isFile && file.name.endsWith(".crash") }
.orEmpty()
.mapNotNull { file ->
val report = runCatching {
CrashReportCodec.decode(file.readText(StandardCharsets.UTF_8))
}.getOrNull()
if (report == null) {
file.delete()
null
} else {
file to report
}
}
.sortedByDescending { (_, report) -> report.timestampMillis }
reports.forEachIndexed { index, (file, report) ->
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) file.delete()
}
}
} }

View File

@@ -0,0 +1,37 @@
package com.vnidrop.app.diagnostics
import java.io.BufferedReader
import java.io.InputStreamReader
import java.net.HttpURLConnection
import java.net.URI
import java.nio.charset.StandardCharsets
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
actual suspend fun platformHttpPost(
url: String,
headers: Map<String, String>,
bodyUtf8: String,
): PlatformHttpResponse = withContext(Dispatchers.IO) {
val connection = (URI(url).toURL().openConnection() as HttpURLConnection).apply {
requestMethod = "POST"
doOutput = true
connectTimeout = 15_000
readTimeout = 30_000
setRequestProperty("Content-Type", "application/json; charset=utf-8")
headers.forEach { (key, value) -> setRequestProperty(key, value) }
}
try {
connection.outputStream.use { output ->
output.write(bodyUtf8.toByteArray(StandardCharsets.UTF_8))
}
val code = connection.responseCode
val stream = if (code in 200..299) connection.inputStream else connection.errorStream
val body = stream?.use { input ->
BufferedReader(InputStreamReader(input, StandardCharsets.UTF_8)).readText()
}.orEmpty()
PlatformHttpResponse(code, body)
} finally {
connection.disconnect()
}
}

View File

@@ -0,0 +1,56 @@
package com.vnidrop.app.diagnostics
import java.io.File
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class PendingCrashStoreJvmTest {
@Test
fun replacesReportsAndPrunesOldCorruptAndTemporaryFiles() {
val root = Files.createTempDirectory("vnidrop-crash-store").toFile()
try {
val store = createPendingCrashStore(root.absolutePath)
val older = report("10000000-0000-4000-8000-000000000001", 1, "older")
val current = report("10000000-0000-4000-8000-000000000002", 2, "current")
store.write(older)
store.write(current)
store.write(current.copy(exceptionMessage = "replaced"))
val directory = File(root, "diagnostics/crashes")
File(directory, "corrupt.crash").writeText("not a crash envelope")
File(directory, ".orphan.tmp").writeText("partial")
store.write(current.copy(id = "../../escape"))
val escapedPath = File(directory, "../../escape.crash").canonicalFile
assertEquals(
listOf("replaced", "older"),
store.list().map(CrashReport::exceptionMessage),
)
store.prune(olderThanTimestampMillis = 0, maxCount = 1)
assertEquals(listOf("replaced"), store.list().map(CrashReport::exceptionMessage))
assertFalse(File(directory, "corrupt.crash").exists())
assertFalse(File(directory, ".orphan.tmp").exists())
assertFalse(escapedPath.exists())
assertTrue(directory.listFiles().orEmpty().all { it.parentFile == directory })
} finally {
root.deleteRecursively()
}
}
private fun report(id: String, timestampMillis: Long, message: String) = CrashReport(
id = id,
timestampMillis = timestampMillis,
installId = "install",
appVersion = "1.0",
platform = "Desktop",
exceptionType = "TestError",
exceptionMessage = message,
stackTrace = "stack",
breadcrumbs = emptyList(),
diagnosticsEnabledAtCapture = true,
)
}