mirror of
https://github.com/sudosylabs/vnidrop.git
synced 2026-08-05 02:29:55 +02:00
feat(diagnostics): harden ingestion and delivery
This commit is contained in:
53
.github/workflows/diagnostics-api.yml
vendored
Normal file
53
.github/workflows/diagnostics-api.yml
vendored
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
name: Diagnostics API
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "services/diagnostics-api/**"
|
||||||
|
- ".github/workflows/diagnostics-api.yml"
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths:
|
||||||
|
- "services/diagnostics-api/**"
|
||||||
|
- ".github/workflows/diagnostics-api.yml"
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: diagnostics-api-${{ github.event.pull_request.number || github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
quality:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: services/diagnostics-api
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: services/diagnostics-api/package-lock.json
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Verify generated Worker types
|
||||||
|
run: npm run types:check
|
||||||
|
|
||||||
|
- name: Type-check
|
||||||
|
run: npm run typecheck
|
||||||
|
|
||||||
|
- name: Test in the Workers runtime
|
||||||
|
run: npm test
|
||||||
|
|
||||||
|
- name: Validate the deployment bundle
|
||||||
|
run: npm run deploy:dry-run
|
||||||
@@ -18,3 +18,8 @@ android.useAndroidX=true
|
|||||||
# Bug report UI remains available (user-initiated).
|
# Bug report UI remains available (user-initiated).
|
||||||
# Override per build: ./gradlew … -Pvnidrop.diagnostics.included=false
|
# Override per build: ./gradlew … -Pvnidrop.diagnostics.included=false
|
||||||
vnidrop.diagnostics.included=true
|
vnidrop.diagnostics.included=true
|
||||||
|
# Cloudflare Worker base URL (no trailing slash). Both endpoint/key empty → NoOp transport.
|
||||||
|
# Example: https://vnidrop-diagnostics.<subdomain>.workers.dev
|
||||||
|
vnidrop.diagnostics.endpoint=
|
||||||
|
# Shared ingest key (must match Worker secret INGEST_KEY). Keep blank in VCS; use user-level/CI properties.
|
||||||
|
vnidrop.diagnostics.ingestKey=
|
||||||
|
|||||||
6
services/diagnostics-api/.gitignore
vendored
Normal file
6
services/diagnostics-api/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
node_modules/
|
||||||
|
.wrangler/
|
||||||
|
dist/
|
||||||
|
.dev.vars*
|
||||||
|
.env*
|
||||||
|
*.log
|
||||||
196
services/diagnostics-api/README.md
Normal file
196
services/diagnostics-api/README.md
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
# VniDrop diagnostics API
|
||||||
|
|
||||||
|
Cloudflare Worker for ingesting batched telemetry, crash reports, and user-submitted
|
||||||
|
bug reports. D1 stores searchable metadata; R2 stores larger stack traces and logs.
|
||||||
|
|
||||||
|
The service is designed for modest traffic and low operating cost:
|
||||||
|
|
||||||
|
- one D1 row is written per telemetry batch, not per event;
|
||||||
|
- crash stacks and bug logs are stored in R2 instead of D1;
|
||||||
|
- request and batch limits reject oversized work before storage writes;
|
||||||
|
- an hourly scheduled cleanup and an R2 lifecycle rule enforce retention;
|
||||||
|
- no Queue, Durable Object, or KV resources are required.
|
||||||
|
|
||||||
|
Cloudflare quotas and prices change over time. Check the current
|
||||||
|
[Workers pricing](https://developers.cloudflare.com/workers/platform/pricing/),
|
||||||
|
[D1 pricing](https://developers.cloudflare.com/d1/platform/pricing/), and
|
||||||
|
[R2 pricing](https://developers.cloudflare.com/r2/pricing/) before relying on a
|
||||||
|
particular free-plan capacity.
|
||||||
|
|
||||||
|
## API
|
||||||
|
|
||||||
|
All ingest routes require:
|
||||||
|
|
||||||
|
```http
|
||||||
|
X-VniDrop-Key: <INGEST_KEY>
|
||||||
|
```
|
||||||
|
|
||||||
|
The client also sends its anonymous installation ID:
|
||||||
|
|
||||||
|
```http
|
||||||
|
X-VniDrop-Install-Id: <anonymous install UUID>
|
||||||
|
```
|
||||||
|
|
||||||
|
| Method | Path | Body |
|
||||||
|
|--------|------|------|
|
||||||
|
| `GET` | `/live` | process liveness; does not touch storage |
|
||||||
|
| `GET` | `/health` | authenticated readiness; checks required configuration and the D1 schema |
|
||||||
|
| `POST` | `/v1/events` | `{ batchId, installId, appVersion?, platform?, events: [...] }` |
|
||||||
|
| `POST` | `/v1/crashes` | app crash payload |
|
||||||
|
| `POST` | `/v1/bugs` | app bug-report payload |
|
||||||
|
|
||||||
|
Batch and report IDs are client-generated UUIDs. A client must reuse the same ID
|
||||||
|
when retrying so D1 can acknowledge the request without storing it twice.
|
||||||
|
|
||||||
|
Accepted reports return `202`. Defaults are a 262,144-byte request limit and at
|
||||||
|
most 50 events per batch. Cloudflare rate-limit bindings allow 30 requests per
|
||||||
|
installation and 120 requests per source, per ingest route, per minute. Source
|
||||||
|
limits run before shared-key verification so rejected traffic is bounded too.
|
||||||
|
These counters are eventually consistent and local to a Cloudflare location, so
|
||||||
|
they are abuse mitigation rather than billing or authorization controls.
|
||||||
|
|
||||||
|
The API is consumed by native Android, iOS, and desktop clients and does not
|
||||||
|
enable cross-origin browser access. If a browser-based client is added later,
|
||||||
|
define a narrow origin allowlist instead of enabling wildcard CORS.
|
||||||
|
|
||||||
|
`/health` requires `X-VniDrop-Key` and uses the source limiter because it performs
|
||||||
|
D1 reads. `/live` is the only unauthenticated probe and never touches storage.
|
||||||
|
|
||||||
|
## Security model
|
||||||
|
|
||||||
|
`INGEST_KEY` fails closed when it is missing, but it is a shared value embedded in
|
||||||
|
released app binaries. It can be extracted and therefore is **not** user
|
||||||
|
authentication, a durable secret, or sufficient abuse protection by itself.
|
||||||
|
|
||||||
|
- Store the Worker value with `wrangler secret put`; never put it in
|
||||||
|
`wrangler.jsonc`, source control, logs, or command arguments.
|
||||||
|
- Rotate the key when it is exposed and ship the matching app configuration.
|
||||||
|
- Keep the two rate-limit namespaces unique within the Cloudflare account. A
|
||||||
|
namespace reused by another Worker shares counters with it.
|
||||||
|
- Use Cloudflare WAF or account-level rate-limiting rules if public abuse exceeds
|
||||||
|
what the Worker bindings can absorb.
|
||||||
|
- Do not log request bodies. Bug reports can contain contact details and attached
|
||||||
|
logs.
|
||||||
|
|
||||||
|
## Provision and deploy
|
||||||
|
|
||||||
|
Run these commands from this directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm ci
|
||||||
|
npx wrangler login
|
||||||
|
|
||||||
|
npx wrangler d1 create vnidrop-diagnostics
|
||||||
|
npx wrangler r2 bucket create vnidrop-diagnostics
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace the placeholder `database_id` in `wrangler.jsonc` with the UUID returned
|
||||||
|
by `wrangler d1 create`. Set the ingest key interactively, apply the tracked D1
|
||||||
|
migrations, and configure the R2 retention rule once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx wrangler secret put INGEST_KEY
|
||||||
|
npm run db:migrate:remote
|
||||||
|
npx wrangler r2 bucket lifecycle add vnidrop-diagnostics diagnostics-retention --expire-days 90
|
||||||
|
npm run check
|
||||||
|
npm run deploy
|
||||||
|
```
|
||||||
|
|
||||||
|
`npm run deploy` also runs the complete `check` script automatically before
|
||||||
|
Wrangler changes the remote Worker.
|
||||||
|
|
||||||
|
The lifecycle command changes the remote bucket. Before adding or changing a
|
||||||
|
rule, inspect the current state with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx wrangler r2 bucket lifecycle list vnidrop-diagnostics
|
||||||
|
```
|
||||||
|
|
||||||
|
## Local development
|
||||||
|
|
||||||
|
Create an ignored `.dev.vars` file containing a development-only key:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
INGEST_KEY=local-development-only
|
||||||
|
```
|
||||||
|
|
||||||
|
Then initialize the local D1 database and run the Worker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run db:migrate:local
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Wrangler keeps local D1 and R2 state under the ignored `.wrangler/` directory.
|
||||||
|
Use `wrangler dev --test-scheduled` when exercising the hourly cleanup handler.
|
||||||
|
|
||||||
|
## Migrations and generated types
|
||||||
|
|
||||||
|
D1 migrations live in `migrations/` and are recorded in D1's migration ledger.
|
||||||
|
Never edit an applied migration; add the next numbered SQL file instead.
|
||||||
|
|
||||||
|
`worker-configuration.d.ts` is generated from `wrangler.jsonc` and committed so
|
||||||
|
bindings cannot silently drift from the Worker code:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run typegen # regenerate after changing bindings or vars
|
||||||
|
npm run types:check # verify the committed file is current
|
||||||
|
```
|
||||||
|
|
||||||
|
Secrets and optional, commented-out bindings are not generated. The source adds
|
||||||
|
only those narrow extensions to the generated environment type.
|
||||||
|
|
||||||
|
Vitest runs inside the Workers runtime. Its setup applies the same numbered D1
|
||||||
|
migrations to the isolated local database assigned to each test file.
|
||||||
|
|
||||||
|
## Retention
|
||||||
|
|
||||||
|
`RETENTION_DAYS` defaults to 90. The `17 * * * *` cron trigger runs cleanup at
|
||||||
|
17 minutes past every hour. Cleanup works in bounded batches: it deletes each
|
||||||
|
expired report's referenced R2 object before deleting that exact D1 row. The R2
|
||||||
|
lifecycle rule is an independent backstop for stack and log objects, including
|
||||||
|
objects left behind by a partial ingest failure. Each scheduled run can remove
|
||||||
|
8,000 event batches and 7,200 rows from each report table while staying below
|
||||||
|
D1's per-invocation query ceiling. Later hourly runs continue any backlog.
|
||||||
|
Reaching the cap emits a structured warning with the remaining expired-row counts;
|
||||||
|
alert on that warning because
|
||||||
|
retention is necessarily best-effort during sustained distributed abuse.
|
||||||
|
|
||||||
|
The Worker variable and bucket lifecycle are separate configuration surfaces.
|
||||||
|
When changing retention, update both `RETENTION_DAYS` and the R2 lifecycle rule;
|
||||||
|
changing one does not update the other. Cloudflare may delete expired R2 objects
|
||||||
|
after the exact expiration time rather than synchronously at it.
|
||||||
|
|
||||||
|
## App wiring
|
||||||
|
|
||||||
|
Keep the tracked root defaults empty. Configure release builds through the
|
||||||
|
user-level `~/.gradle/gradle.properties` or secured CI Gradle project properties:
|
||||||
|
|
||||||
|
```properties
|
||||||
|
vnidrop.diagnostics.included=true
|
||||||
|
vnidrop.diagnostics.endpoint=https://vnidrop-diagnostics.<your-subdomain>.workers.dev
|
||||||
|
vnidrop.diagnostics.ingestKey=<same value as INGEST_KEY>
|
||||||
|
```
|
||||||
|
|
||||||
|
Both the endpoint and key are required. When both are empty the app uses its
|
||||||
|
offline-safe no-op transport; configuring only one fails the Gradle build.
|
||||||
|
`vnidrop.diagnostics.included=false` disables
|
||||||
|
automatic telemetry and crash upload, but a configured endpoint can still accept
|
||||||
|
an explicit user-submitted bug report. Treat the app-side key as an abuse-control
|
||||||
|
token with the limitations described above.
|
||||||
|
|
||||||
|
## Reading reports
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx wrangler d1 execute vnidrop-diagnostics --remote \
|
||||||
|
--command "SELECT id, exception_type, platform, occurred_at FROM crashes ORDER BY occurred_at DESC LIMIT 20"
|
||||||
|
|
||||||
|
npx wrangler d1 execute vnidrop-diagnostics --remote \
|
||||||
|
--command "SELECT id, what_happened, status, occurred_at FROM bugs WHERE status = 'open' ORDER BY occurred_at DESC LIMIT 20"
|
||||||
|
```
|
||||||
|
|
||||||
|
R2 object keys use `crashes/<id>/<attempt-id>/stack.txt` and
|
||||||
|
`bugs/<id>/<attempt-id>/logs.txt`. The unique attempt segment prevents a retry
|
||||||
|
from overwriting an already accepted object before D1 detects the duplicate.
|
||||||
|
There is no public administration endpoint; inspect reports through authenticated
|
||||||
|
Cloudflare tools or a future Access-protected dashboard.
|
||||||
55
services/diagnostics-api/migrations/0001_initial.sql
Normal file
55
services/diagnostics-api/migrations/0001_initial.sql
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
-- This migration also baselines databases previously initialized by schema.sql.
|
||||||
|
PRAGMA foreign_keys = ON;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS event_batches (
|
||||||
|
id TEXT PRIMARY KEY NOT NULL,
|
||||||
|
received_at INTEGER NOT NULL,
|
||||||
|
install_id TEXT NOT NULL,
|
||||||
|
app_version TEXT,
|
||||||
|
platform TEXT,
|
||||||
|
event_count INTEGER NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_event_batches_received ON event_batches (received_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_event_batches_install ON event_batches (install_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS crashes (
|
||||||
|
id TEXT PRIMARY KEY NOT NULL,
|
||||||
|
received_at INTEGER NOT NULL,
|
||||||
|
occurred_at INTEGER NOT NULL,
|
||||||
|
install_id TEXT NOT NULL,
|
||||||
|
app_version TEXT,
|
||||||
|
platform TEXT,
|
||||||
|
exception_type TEXT,
|
||||||
|
exception_message TEXT,
|
||||||
|
fingerprint TEXT NOT NULL,
|
||||||
|
diagnostics_enabled INTEGER NOT NULL DEFAULT 0,
|
||||||
|
stack_r2_key TEXT,
|
||||||
|
breadcrumbs_json TEXT,
|
||||||
|
schema_version INTEGER NOT NULL DEFAULT 1
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_crashes_received ON crashes (received_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_crashes_fingerprint ON crashes (fingerprint);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_crashes_install ON crashes (install_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS bugs (
|
||||||
|
id TEXT PRIMARY KEY NOT NULL,
|
||||||
|
received_at INTEGER NOT NULL,
|
||||||
|
install_id TEXT NOT NULL,
|
||||||
|
app_version TEXT,
|
||||||
|
platform TEXT,
|
||||||
|
what_happened TEXT NOT NULL,
|
||||||
|
expected TEXT NOT NULL,
|
||||||
|
steps TEXT,
|
||||||
|
contact TEXT,
|
||||||
|
logs_r2_key TEXT,
|
||||||
|
device_json TEXT,
|
||||||
|
breadcrumbs_json TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'open',
|
||||||
|
schema_version INTEGER NOT NULL DEFAULT 1
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_bugs_received ON bugs (received_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_bugs_status ON bugs (status);
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
ALTER TABLE bugs ADD COLUMN occurred_at INTEGER;
|
||||||
|
|
||||||
|
-- Existing reports predate this field; their receipt time is the best available value.
|
||||||
|
UPDATE bugs SET occurred_at = received_at WHERE occurred_at IS NULL;
|
||||||
3151
services/diagnostics-api/package-lock.json
generated
Normal file
3151
services/diagnostics-api/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
services/diagnostics-api/package.json
Normal file
30
services/diagnostics-api/package.json
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"name": "vnidrop-diagnostics-api",
|
||||||
|
"private": true,
|
||||||
|
"version": "0.1.0",
|
||||||
|
"type": "module",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=22.12"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"dev": "wrangler dev",
|
||||||
|
"predeploy": "npm run check",
|
||||||
|
"deploy": "wrangler deploy",
|
||||||
|
"deploy:dry-run": "wrangler deploy --dry-run",
|
||||||
|
"db:migrate:local": "wrangler d1 migrations apply vnidrop-diagnostics --local",
|
||||||
|
"db:migrate:remote": "wrangler d1 migrations apply vnidrop-diagnostics --remote",
|
||||||
|
"test": "vitest run",
|
||||||
|
"test:watch": "vitest",
|
||||||
|
"typegen": "wrangler types",
|
||||||
|
"types:check": "wrangler types --check",
|
||||||
|
"typecheck": "tsc --noEmit && tsc --noEmit -p test/tsconfig.json",
|
||||||
|
"check": "npm run types:check && npm run typecheck && npm test && npm run deploy:dry-run"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@cloudflare/vitest-pool-workers": "^0.18.4",
|
||||||
|
"@types/node": "^26.1.1",
|
||||||
|
"typescript": "^7.0.2",
|
||||||
|
"vitest": "^4.1.10",
|
||||||
|
"wrangler": "^4.110.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
267
services/diagnostics-api/src/index.ts
Normal file
267
services/diagnostics-api/src/index.ts
Normal file
@@ -0,0 +1,267 @@
|
|||||||
|
import {
|
||||||
|
normalizeBug,
|
||||||
|
normalizeCrash,
|
||||||
|
normalizeEvents,
|
||||||
|
readJsonObject,
|
||||||
|
} from "./input";
|
||||||
|
import {
|
||||||
|
type DiagnosticsEnv,
|
||||||
|
runRetention,
|
||||||
|
storeBug,
|
||||||
|
storeCrash,
|
||||||
|
storeEvents,
|
||||||
|
} from "./storage";
|
||||||
|
|
||||||
|
const DEFAULT_MAX_BODY_BYTES = 262_144;
|
||||||
|
const HARD_MAX_BODY_BYTES = 1_048_576;
|
||||||
|
const DEFAULT_MAX_EVENTS = 50;
|
||||||
|
|
||||||
|
export default {
|
||||||
|
async fetch(request: Request, env: DiagnosticsEnv, _ctx: ExecutionContext): Promise<Response> {
|
||||||
|
const requestId = crypto.randomUUID();
|
||||||
|
const url = new URL(request.url);
|
||||||
|
try {
|
||||||
|
if (request.method === "GET" && url.pathname === "/live") {
|
||||||
|
return json({ ok: true, service: "vnidrop-diagnostics", schema: 1 }, 200, requestId);
|
||||||
|
}
|
||||||
|
if (request.method === "GET" && url.pathname === "/health") {
|
||||||
|
if (await sourceRateLimited(request, url.pathname, env)) {
|
||||||
|
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
|
||||||
|
}
|
||||||
|
const authError = await authorize(request, env);
|
||||||
|
if (authError) return json({ error: authError.error }, authError.status, requestId);
|
||||||
|
return await readiness(env, requestId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isIngestPath(url.pathname)) {
|
||||||
|
return json({ error: "not_found" }, 404, requestId);
|
||||||
|
}
|
||||||
|
if (request.method === "OPTIONS") {
|
||||||
|
return new Response(null, {
|
||||||
|
status: 204,
|
||||||
|
headers: responseHeaders(requestId, { allow: "POST, OPTIONS" }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (request.method !== "POST") {
|
||||||
|
return json(
|
||||||
|
{ error: "method_not_allowed" },
|
||||||
|
405,
|
||||||
|
requestId,
|
||||||
|
{ allow: "POST, OPTIONS" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (await sourceRateLimited(request, url.pathname, env)) {
|
||||||
|
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const authError = await authorize(request, env);
|
||||||
|
if (authError) return json({ error: authError.error }, authError.status, requestId);
|
||||||
|
|
||||||
|
if (await installRateLimited(request, url.pathname, env)) {
|
||||||
|
return json({ error: "rate_limited" }, 429, requestId, { "retry-after": "60" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxBodyBytes = boundedPositiveInt(
|
||||||
|
env.MAX_BODY_BYTES,
|
||||||
|
DEFAULT_MAX_BODY_BYTES,
|
||||||
|
1,
|
||||||
|
HARD_MAX_BODY_BYTES,
|
||||||
|
);
|
||||||
|
const parsed = await readJsonObject(request, maxBodyBytes);
|
||||||
|
if (!parsed.ok) return json({ error: parsed.error }, parsed.status, requestId);
|
||||||
|
|
||||||
|
switch (url.pathname) {
|
||||||
|
case "/v1/events": {
|
||||||
|
const maxEvents = boundedPositiveInt(env.MAX_EVENTS_PER_BATCH, DEFAULT_MAX_EVENTS, 1, 100);
|
||||||
|
const normalized = normalizeEvents(parsed.value, maxEvents);
|
||||||
|
if (!normalized.ok) {
|
||||||
|
return json({ error: normalized.error }, normalized.status, requestId);
|
||||||
|
}
|
||||||
|
const result = await storeEvents(normalized.value, env);
|
||||||
|
return json(
|
||||||
|
{
|
||||||
|
ok: true,
|
||||||
|
id: result.id,
|
||||||
|
stored: result.stored,
|
||||||
|
duplicate: result.duplicate,
|
||||||
|
},
|
||||||
|
202,
|
||||||
|
requestId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case "/v1/crashes": {
|
||||||
|
const normalized = normalizeCrash(parsed.value);
|
||||||
|
if (!normalized.ok) {
|
||||||
|
return json({ error: normalized.error }, normalized.status, requestId);
|
||||||
|
}
|
||||||
|
const result = await storeCrash(normalized.value, env);
|
||||||
|
return json(
|
||||||
|
{
|
||||||
|
ok: true,
|
||||||
|
id: result.id,
|
||||||
|
fingerprint: result.fingerprint,
|
||||||
|
duplicate: result.duplicate,
|
||||||
|
},
|
||||||
|
202,
|
||||||
|
requestId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case "/v1/bugs": {
|
||||||
|
const normalized = normalizeBug(parsed.value);
|
||||||
|
if (!normalized.ok) {
|
||||||
|
return json({ error: normalized.error }, normalized.status, requestId);
|
||||||
|
}
|
||||||
|
const result = await storeBug(normalized.value, env);
|
||||||
|
return json(
|
||||||
|
{ ok: true, id: result.id, duplicate: result.duplicate },
|
||||||
|
202,
|
||||||
|
requestId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "diagnostics request failed",
|
||||||
|
requestId,
|
||||||
|
method: request.method,
|
||||||
|
path: url.pathname,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return json({ error: "internal" }, 500, requestId);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
scheduled(
|
||||||
|
controller: ScheduledController,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
ctx: ExecutionContext,
|
||||||
|
): void {
|
||||||
|
ctx.waitUntil(
|
||||||
|
runRetention(env).catch((error) => {
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "diagnostics retention failed",
|
||||||
|
scheduledTime: controller.scheduledTime,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
} satisfies ExportedHandler<DiagnosticsEnv>;
|
||||||
|
|
||||||
|
async function readiness(env: DiagnosticsEnv, requestId: string): Promise<Response> {
|
||||||
|
if (!env.INGEST_KEY) {
|
||||||
|
return json({ ok: false, error: "server_misconfigured" }, 503, requestId);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await env.DB.batch([
|
||||||
|
env.DB.prepare(
|
||||||
|
"SELECT id, received_at, install_id, payload_json FROM event_batches LIMIT 1",
|
||||||
|
),
|
||||||
|
env.DB.prepare(
|
||||||
|
"SELECT id, occurred_at, stack_r2_key, breadcrumbs_json FROM crashes LIMIT 1",
|
||||||
|
),
|
||||||
|
env.DB.prepare(
|
||||||
|
"SELECT id, occurred_at, logs_r2_key, device_json FROM bugs LIMIT 1",
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
return json({ ok: true, service: "vnidrop-diagnostics", schema: 1 }, 200, requestId);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "diagnostics readiness check failed",
|
||||||
|
requestId,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return json({ ok: false, error: "dependency_unavailable" }, 503, requestId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function authorize(
|
||||||
|
request: Request,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<{ status: 401 | 503; error: "unauthorized" | "server_misconfigured" } | null> {
|
||||||
|
const expected = env.INGEST_KEY;
|
||||||
|
if (!expected) return { status: 503, error: "server_misconfigured" };
|
||||||
|
const provided = request.headers.get("x-vnidrop-key") ?? "";
|
||||||
|
if (!(await timingSafeEqual(provided, expected))) {
|
||||||
|
return { status: 401, error: "unauthorized" };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sourceRateLimited(
|
||||||
|
request: Request,
|
||||||
|
path: string,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const source = request.headers.get("cf-connecting-ip")?.slice(0, 64) || "unknown";
|
||||||
|
const result = await env.SOURCE_RATE_LIMITER.limit({ key: `${source}:${path}` });
|
||||||
|
return !result.success;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function installRateLimited(
|
||||||
|
request: Request,
|
||||||
|
path: string,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const source = request.headers.get("cf-connecting-ip")?.slice(0, 64) || "unknown";
|
||||||
|
const installId = request.headers.get("x-vnidrop-install-id")?.trim().slice(0, 80) || source;
|
||||||
|
const result = await env.INSTALL_RATE_LIMITER.limit({ key: `${installId}:${path}` });
|
||||||
|
return !result.success;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isIngestPath(path: string): path is "/v1/events" | "/v1/crashes" | "/v1/bugs" {
|
||||||
|
return path === "/v1/events" || path === "/v1/crashes" || path === "/v1/bugs";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function timingSafeEqual(provided: string, expected: string): Promise<boolean> {
|
||||||
|
const encoder = new TextEncoder();
|
||||||
|
const [providedHash, expectedHash] = await Promise.all([
|
||||||
|
crypto.subtle.digest("SHA-256", encoder.encode(provided)),
|
||||||
|
crypto.subtle.digest("SHA-256", encoder.encode(expected)),
|
||||||
|
]);
|
||||||
|
return crypto.subtle.timingSafeEqual(providedHash, expectedHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
function json(
|
||||||
|
body: unknown,
|
||||||
|
status: number,
|
||||||
|
requestId: string,
|
||||||
|
extraHeaders?: Readonly<Record<string, string>>,
|
||||||
|
): Response {
|
||||||
|
return new Response(JSON.stringify(body), {
|
||||||
|
status,
|
||||||
|
headers: responseHeaders(requestId, {
|
||||||
|
"content-type": "application/json; charset=utf-8",
|
||||||
|
...extraHeaders,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function responseHeaders(
|
||||||
|
requestId: string,
|
||||||
|
extraHeaders?: Readonly<Record<string, string>>,
|
||||||
|
): Headers {
|
||||||
|
const headers = new Headers(extraHeaders);
|
||||||
|
headers.set("cache-control", "no-store");
|
||||||
|
headers.set("x-content-type-options", "nosniff");
|
||||||
|
headers.set("x-request-id", requestId);
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
function boundedPositiveInt(
|
||||||
|
raw: string | undefined,
|
||||||
|
fallback: number,
|
||||||
|
minimum: number,
|
||||||
|
maximum: number,
|
||||||
|
): number {
|
||||||
|
const parsed = Number(raw);
|
||||||
|
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) return fallback;
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
593
services/diagnostics-api/src/input.ts
Normal file
593
services/diagnostics-api/src/input.ts
Normal file
@@ -0,0 +1,593 @@
|
|||||||
|
export type JsonObject = Record<string, unknown>;
|
||||||
|
|
||||||
|
export type InputFailure = {
|
||||||
|
ok: false;
|
||||||
|
status: 400 | 413 | 415;
|
||||||
|
error: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InputResult<T> = { ok: true; value: T } | InputFailure;
|
||||||
|
|
||||||
|
export type NormalizedProperties = Record<string, string>;
|
||||||
|
|
||||||
|
export interface NormalizedEvent {
|
||||||
|
name: string;
|
||||||
|
timestampMillis: number;
|
||||||
|
properties: NormalizedProperties;
|
||||||
|
schemaVersion: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NormalizedBreadcrumb {
|
||||||
|
name: string;
|
||||||
|
timestampMillis: number;
|
||||||
|
properties: NormalizedProperties;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NormalizedDevice {
|
||||||
|
deviceName: string;
|
||||||
|
deviceModel: string;
|
||||||
|
operatingSystem: string;
|
||||||
|
network: string;
|
||||||
|
batteryLevel: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NormalizedEventsPayload {
|
||||||
|
batchId: string;
|
||||||
|
installId: string;
|
||||||
|
appVersion: string;
|
||||||
|
platform: string;
|
||||||
|
events: NormalizedEvent[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NormalizedCrashPayload {
|
||||||
|
id: string;
|
||||||
|
installId: string;
|
||||||
|
appVersion: string;
|
||||||
|
platform: string;
|
||||||
|
exceptionType: string;
|
||||||
|
exceptionMessage: string;
|
||||||
|
stackTrace: string;
|
||||||
|
occurredAt: number;
|
||||||
|
diagnosticsEnabledAtCapture: boolean;
|
||||||
|
breadcrumbs: NormalizedBreadcrumb[];
|
||||||
|
schemaVersion: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NormalizedBugPayload {
|
||||||
|
id: string;
|
||||||
|
installId: string;
|
||||||
|
appVersion: string;
|
||||||
|
platform: string;
|
||||||
|
occurredAt: number;
|
||||||
|
whatHappened: string;
|
||||||
|
expected: string;
|
||||||
|
steps: string;
|
||||||
|
contact: string;
|
||||||
|
logs: string;
|
||||||
|
device: NormalizedDevice;
|
||||||
|
breadcrumbs: NormalizedBreadcrumb[];
|
||||||
|
schemaVersion: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const MAX_LOG_BYTES = 192 * 1024;
|
||||||
|
export const MAX_BREADCRUMBS_JSON_BYTES = 16_000;
|
||||||
|
export const MAX_DEVICE_JSON_BYTES = 4_000;
|
||||||
|
|
||||||
|
const MAX_PROPERTIES = 12;
|
||||||
|
const MAX_BREADCRUMBS = 40;
|
||||||
|
const MISSING = Symbol("missing");
|
||||||
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||||
|
const UTF8_ENCODER = new TextEncoder();
|
||||||
|
|
||||||
|
export async function readJsonObject(
|
||||||
|
request: Request,
|
||||||
|
maxBytes: number,
|
||||||
|
): Promise<InputResult<JsonObject>> {
|
||||||
|
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
|
||||||
|
throw new RangeError("maxBytes must be a positive safe integer");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isApplicationJson(request.headers.get("content-type"))) {
|
||||||
|
return failure(415, "unsupported_media_type");
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = request.headers.get("content-length");
|
||||||
|
if (contentLength != null) {
|
||||||
|
const trimmed = contentLength.trim();
|
||||||
|
if (!/^\d+$/.test(trimmed)) {
|
||||||
|
return failure(400, "invalid_content_length");
|
||||||
|
}
|
||||||
|
const normalizedLength = trimmed.replace(/^0+(?=\d)/, "");
|
||||||
|
const maxLength = String(maxBytes);
|
||||||
|
if (
|
||||||
|
normalizedLength.length > maxLength.length ||
|
||||||
|
(normalizedLength.length === maxLength.length && normalizedLength > maxLength)
|
||||||
|
) {
|
||||||
|
return failure(413, "payload_too_large");
|
||||||
|
}
|
||||||
|
const declaredBytes = Number(trimmed);
|
||||||
|
if (!Number.isSafeInteger(declaredBytes)) {
|
||||||
|
return failure(400, "invalid_content_length");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (request.body == null) {
|
||||||
|
return failure(400, "invalid_json");
|
||||||
|
}
|
||||||
|
|
||||||
|
let reader: ReadableStreamDefaultReader<Uint8Array>;
|
||||||
|
try {
|
||||||
|
reader = request.body.getReader();
|
||||||
|
} catch {
|
||||||
|
return failure(400, "invalid_body");
|
||||||
|
}
|
||||||
|
const chunks: Uint8Array[] = [];
|
||||||
|
let totalBytes = 0;
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const chunk = await reader.read();
|
||||||
|
if (chunk.done) break;
|
||||||
|
if (chunk.value.byteLength > maxBytes - totalBytes) {
|
||||||
|
try {
|
||||||
|
await reader.cancel("payload_too_large");
|
||||||
|
} catch {
|
||||||
|
// The size failure remains authoritative if cancellation also fails.
|
||||||
|
}
|
||||||
|
return failure(413, "payload_too_large");
|
||||||
|
}
|
||||||
|
totalBytes += chunk.value.byteLength;
|
||||||
|
chunks.push(chunk.value);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return failure(400, "invalid_body");
|
||||||
|
} finally {
|
||||||
|
reader.releaseLock();
|
||||||
|
}
|
||||||
|
|
||||||
|
const bytes = joinChunks(chunks, totalBytes);
|
||||||
|
let raw: string;
|
||||||
|
try {
|
||||||
|
raw = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
||||||
|
} catch {
|
||||||
|
return failure(400, "invalid_utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
return failure(400, "invalid_json");
|
||||||
|
}
|
||||||
|
if (!isPlainObject(parsed)) {
|
||||||
|
return failure(400, "invalid_body");
|
||||||
|
}
|
||||||
|
return success(parsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeEvents(
|
||||||
|
body: JsonObject,
|
||||||
|
maxEvents = 50,
|
||||||
|
): InputResult<NormalizedEventsPayload> {
|
||||||
|
if (!isPlainObject(body)) return failure(400, "invalid_body");
|
||||||
|
if (!Number.isSafeInteger(maxEvents) || maxEvents <= 0) {
|
||||||
|
throw new RangeError("maxEvents must be a positive safe integer");
|
||||||
|
}
|
||||||
|
|
||||||
|
const batchId = idField(body, ["batchId", "batch_id"], "invalid_batch_id");
|
||||||
|
if (!batchId.ok) return batchId;
|
||||||
|
const installId = installIdField(body);
|
||||||
|
if (!installId.ok) return installId;
|
||||||
|
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
|
||||||
|
if (!appVersion.ok) return appVersion;
|
||||||
|
const platform = stringField(body, ["platform"], 40, "invalid_platform");
|
||||||
|
if (!platform.ok) return platform;
|
||||||
|
|
||||||
|
const batchSchema = schemaVersion(body);
|
||||||
|
if (!batchSchema.ok) return batchSchema;
|
||||||
|
const rawEvents = pick(body, ["events"]);
|
||||||
|
if (!Array.isArray(rawEvents)) return failure(400, "invalid_events");
|
||||||
|
if (rawEvents.length === 0) return failure(400, "empty_batch");
|
||||||
|
if (rawEvents.length > maxEvents) return failure(400, "batch_too_large");
|
||||||
|
|
||||||
|
const events: NormalizedEvent[] = [];
|
||||||
|
for (const rawEvent of rawEvents) {
|
||||||
|
const event = normalizeEvent(rawEvent);
|
||||||
|
if (!event.ok) return event;
|
||||||
|
events.push(event.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
return success({
|
||||||
|
batchId: batchId.value,
|
||||||
|
installId: installId.value,
|
||||||
|
appVersion: appVersion.value,
|
||||||
|
platform: platform.value,
|
||||||
|
events,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeCrash(body: JsonObject): InputResult<NormalizedCrashPayload> {
|
||||||
|
if (!isPlainObject(body)) return failure(400, "invalid_body");
|
||||||
|
|
||||||
|
const id = idField(body, ["id"], "invalid_id");
|
||||||
|
if (!id.ok) return id;
|
||||||
|
const installId = installIdField(body);
|
||||||
|
if (!installId.ok) return installId;
|
||||||
|
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
|
||||||
|
if (!appVersion.ok) return appVersion;
|
||||||
|
const platform = stringField(body, ["platform"], 40, "invalid_platform");
|
||||||
|
if (!platform.ok) return platform;
|
||||||
|
const exceptionType = stringField(
|
||||||
|
body,
|
||||||
|
["exceptionType", "exception_type"],
|
||||||
|
120,
|
||||||
|
"invalid_exception_type",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!exceptionType.ok) return exceptionType;
|
||||||
|
const exceptionMessage = stringField(
|
||||||
|
body,
|
||||||
|
["exceptionMessage", "exception_message"],
|
||||||
|
2_000,
|
||||||
|
"invalid_exception_message",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!exceptionMessage.ok) return exceptionMessage;
|
||||||
|
const stackTrace = stringField(
|
||||||
|
body,
|
||||||
|
["stackTrace", "stack_trace"],
|
||||||
|
32_000,
|
||||||
|
"invalid_stack_trace",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!stackTrace.ok) return stackTrace;
|
||||||
|
const occurredAt = timestampField(
|
||||||
|
body,
|
||||||
|
["timestampMillis", "timestamp_millis", "occurredAt", "occurred_at"],
|
||||||
|
);
|
||||||
|
if (!occurredAt.ok) return occurredAt;
|
||||||
|
const diagnosticsEnabled = booleanField(
|
||||||
|
body,
|
||||||
|
[
|
||||||
|
"diagnosticsEnabledAtCapture",
|
||||||
|
"diagnostics_enabled_at_capture",
|
||||||
|
"diagnostics_enabled",
|
||||||
|
],
|
||||||
|
"invalid_diagnostics_enabled",
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!diagnosticsEnabled.ok) return diagnosticsEnabled;
|
||||||
|
const version = schemaVersion(body);
|
||||||
|
if (!version.ok) return version;
|
||||||
|
const breadcrumbs = normalizeBreadcrumbs(pick(body, ["breadcrumbs"]));
|
||||||
|
if (!breadcrumbs.ok) return breadcrumbs;
|
||||||
|
|
||||||
|
return success({
|
||||||
|
id: id.value,
|
||||||
|
installId: installId.value,
|
||||||
|
appVersion: appVersion.value,
|
||||||
|
platform: platform.value,
|
||||||
|
exceptionType: exceptionType.value,
|
||||||
|
exceptionMessage: exceptionMessage.value,
|
||||||
|
stackTrace: stackTrace.value,
|
||||||
|
occurredAt: occurredAt.value,
|
||||||
|
diagnosticsEnabledAtCapture: diagnosticsEnabled.value,
|
||||||
|
breadcrumbs: breadcrumbs.value,
|
||||||
|
schemaVersion: version.value,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeBug(body: JsonObject): InputResult<NormalizedBugPayload> {
|
||||||
|
if (!isPlainObject(body)) return failure(400, "invalid_body");
|
||||||
|
|
||||||
|
const id = idField(body, ["id"], "invalid_id");
|
||||||
|
if (!id.ok) return id;
|
||||||
|
const installId = installIdField(body);
|
||||||
|
if (!installId.ok) return installId;
|
||||||
|
const appVersion = stringField(body, ["appVersion", "app_version"], 40, "invalid_app_version");
|
||||||
|
if (!appVersion.ok) return appVersion;
|
||||||
|
const platform = stringField(body, ["platform"], 40, "invalid_platform");
|
||||||
|
if (!platform.ok) return platform;
|
||||||
|
const occurredAt = timestampField(
|
||||||
|
body,
|
||||||
|
["timestampMillis", "timestamp_millis", "occurredAt", "occurred_at"],
|
||||||
|
);
|
||||||
|
if (!occurredAt.ok) return occurredAt;
|
||||||
|
const whatHappened = stringField(
|
||||||
|
body,
|
||||||
|
["whatHappened", "what_happened"],
|
||||||
|
4_000,
|
||||||
|
"missing_fields",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!whatHappened.ok) return whatHappened;
|
||||||
|
const expected = stringField(body, ["expected"], 4_000, "missing_fields", true, true);
|
||||||
|
if (!expected.ok) return expected;
|
||||||
|
const steps = stringField(body, ["steps"], 4_000, "invalid_steps");
|
||||||
|
if (!steps.ok) return steps;
|
||||||
|
const contact = stringField(body, ["contact"], 320, "invalid_contact");
|
||||||
|
if (!contact.ok) return contact;
|
||||||
|
const includeLogs = booleanField(
|
||||||
|
body,
|
||||||
|
["includeLogs", "include_logs"],
|
||||||
|
"invalid_include_logs",
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
if (!includeLogs.ok) return includeLogs;
|
||||||
|
const logs = stringField(body, ["logs"], MAX_LOG_BYTES, "invalid_logs");
|
||||||
|
if (!logs.ok) return logs;
|
||||||
|
const device = normalizeDevice(pick(body, ["device"]));
|
||||||
|
if (!device.ok) return device;
|
||||||
|
const breadcrumbs = normalizeBreadcrumbs(pick(body, ["breadcrumbs"]));
|
||||||
|
if (!breadcrumbs.ok) return breadcrumbs;
|
||||||
|
const version = schemaVersion(body);
|
||||||
|
if (!version.ok) return version;
|
||||||
|
|
||||||
|
return success({
|
||||||
|
id: id.value,
|
||||||
|
installId: installId.value,
|
||||||
|
appVersion: appVersion.value,
|
||||||
|
platform: platform.value,
|
||||||
|
occurredAt: occurredAt.value,
|
||||||
|
whatHappened: whatHappened.value,
|
||||||
|
expected: expected.value,
|
||||||
|
steps: steps.value,
|
||||||
|
contact: contact.value,
|
||||||
|
logs: includeLogs.value === true ? logs.value : "",
|
||||||
|
device: device.value,
|
||||||
|
breadcrumbs: breadcrumbs.value,
|
||||||
|
schemaVersion: version.value,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeEvent(raw: unknown): InputResult<NormalizedEvent> {
|
||||||
|
if (!isPlainObject(raw)) return failure(400, "invalid_event");
|
||||||
|
const name = stringField(raw, ["name"], 64, "invalid_event", true, true);
|
||||||
|
if (!name.ok) return name;
|
||||||
|
const timestamp = timestampField(raw, ["timestampMillis", "timestamp_millis", "ts"]);
|
||||||
|
if (!timestamp.ok) return failure(400, "invalid_event");
|
||||||
|
const properties = normalizeProperties(pick(raw, ["properties", "props"]), "invalid_event");
|
||||||
|
if (!properties.ok) return properties;
|
||||||
|
const version = schemaVersion(raw);
|
||||||
|
if (!version.ok) return version;
|
||||||
|
return success({
|
||||||
|
name: name.value,
|
||||||
|
timestampMillis: timestamp.value,
|
||||||
|
properties: properties.value,
|
||||||
|
schemaVersion: version.value,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeBreadcrumbs(raw: unknown | typeof MISSING): InputResult<NormalizedBreadcrumb[]> {
|
||||||
|
if (raw === MISSING) return success([]);
|
||||||
|
if (!Array.isArray(raw)) return failure(400, "invalid_breadcrumbs");
|
||||||
|
|
||||||
|
const breadcrumbs: NormalizedBreadcrumb[] = [];
|
||||||
|
for (const item of raw.slice(0, MAX_BREADCRUMBS)) {
|
||||||
|
if (!isPlainObject(item)) return failure(400, "invalid_breadcrumbs");
|
||||||
|
const name = stringField(item, ["name"], 64, "invalid_breadcrumbs", true, true);
|
||||||
|
if (!name.ok) return name;
|
||||||
|
const timestamp = timestampField(item, ["timestampMillis", "timestamp_millis", "ts"]);
|
||||||
|
if (!timestamp.ok) return failure(400, "invalid_breadcrumbs");
|
||||||
|
const properties = normalizeProperties(
|
||||||
|
pick(item, ["properties", "props"]),
|
||||||
|
"invalid_breadcrumbs",
|
||||||
|
);
|
||||||
|
if (!properties.ok) return properties;
|
||||||
|
|
||||||
|
breadcrumbs.push({
|
||||||
|
name: name.value,
|
||||||
|
timestampMillis: timestamp.value,
|
||||||
|
properties: properties.value,
|
||||||
|
});
|
||||||
|
if (jsonBytes(breadcrumbs) > MAX_BREADCRUMBS_JSON_BYTES) {
|
||||||
|
breadcrumbs.pop();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return success(breadcrumbs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeProperties(
|
||||||
|
raw: unknown | typeof MISSING,
|
||||||
|
error: string,
|
||||||
|
): InputResult<NormalizedProperties> {
|
||||||
|
if (raw === MISSING) return success({});
|
||||||
|
if (!isPlainObject(raw)) return failure(400, error);
|
||||||
|
|
||||||
|
const entries: Array<[string, string]> = [];
|
||||||
|
const normalizedKeys = new Set<string>();
|
||||||
|
for (const [key, value] of Object.entries(raw).slice(0, MAX_PROPERTIES)) {
|
||||||
|
if (typeof value !== "string") return failure(400, error);
|
||||||
|
const normalizedKey = truncateUtf8(key, 40);
|
||||||
|
if (normalizedKey.length === 0 || normalizedKeys.has(normalizedKey)) {
|
||||||
|
return failure(400, error);
|
||||||
|
}
|
||||||
|
normalizedKeys.add(normalizedKey);
|
||||||
|
entries.push([normalizedKey, truncateUtf8(value, 128)]);
|
||||||
|
}
|
||||||
|
return success(Object.fromEntries(entries));
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDevice(raw: unknown | typeof MISSING): InputResult<NormalizedDevice> {
|
||||||
|
if (raw === MISSING) raw = {};
|
||||||
|
if (!isPlainObject(raw)) return failure(400, "invalid_device");
|
||||||
|
|
||||||
|
const deviceName = stringField(raw, ["deviceName", "device_name"], 128, "invalid_device");
|
||||||
|
if (!deviceName.ok) return deviceName;
|
||||||
|
const deviceModel = stringField(raw, ["deviceModel", "device_model"], 128, "invalid_device");
|
||||||
|
if (!deviceModel.ok) return deviceModel;
|
||||||
|
const operatingSystem = stringField(
|
||||||
|
raw,
|
||||||
|
["operatingSystem", "operating_system"],
|
||||||
|
192,
|
||||||
|
"invalid_device",
|
||||||
|
);
|
||||||
|
if (!operatingSystem.ok) return operatingSystem;
|
||||||
|
const network = stringField(raw, ["network"], 96, "invalid_device");
|
||||||
|
if (!network.ok) return network;
|
||||||
|
const batteryLevel = stringField(raw, ["batteryLevel", "battery_level"], 64, "invalid_device");
|
||||||
|
if (!batteryLevel.ok) return batteryLevel;
|
||||||
|
|
||||||
|
const device: NormalizedDevice = {
|
||||||
|
deviceName: deviceName.value,
|
||||||
|
deviceModel: deviceModel.value,
|
||||||
|
operatingSystem: operatingSystem.value,
|
||||||
|
network: network.value,
|
||||||
|
batteryLevel: batteryLevel.value,
|
||||||
|
};
|
||||||
|
if (jsonBytes(device) > MAX_DEVICE_JSON_BYTES) {
|
||||||
|
return failure(400, "invalid_device");
|
||||||
|
}
|
||||||
|
return success(device);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringField(
|
||||||
|
object: JsonObject,
|
||||||
|
keys: readonly string[],
|
||||||
|
maxBytes: number,
|
||||||
|
error: string,
|
||||||
|
required = false,
|
||||||
|
nonEmpty = false,
|
||||||
|
): InputResult<string> {
|
||||||
|
const raw = pick(object, keys);
|
||||||
|
if (raw === MISSING) {
|
||||||
|
return required ? failure(400, error) : success("");
|
||||||
|
}
|
||||||
|
if (typeof raw !== "string") return failure(400, error);
|
||||||
|
const value = truncateUtf8(raw, maxBytes);
|
||||||
|
if (nonEmpty && value.trim().length === 0) return failure(400, error);
|
||||||
|
return success(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function idField(
|
||||||
|
object: JsonObject,
|
||||||
|
keys: readonly string[],
|
||||||
|
error: string,
|
||||||
|
): InputResult<string> {
|
||||||
|
const raw = pick(object, keys);
|
||||||
|
if (typeof raw !== "string" || !UUID_PATTERN.test(raw)) return failure(400, error);
|
||||||
|
return success(raw.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
function installIdField(object: JsonObject): InputResult<string> {
|
||||||
|
const raw = pick(object, ["installId", "install_id"]);
|
||||||
|
if (raw === MISSING || raw === "") return success("unknown");
|
||||||
|
if (
|
||||||
|
typeof raw !== "string" ||
|
||||||
|
raw.trim().length === 0 ||
|
||||||
|
/[\u0000-\u001f\u007f]/.test(raw)
|
||||||
|
) {
|
||||||
|
return failure(400, "invalid_install_id");
|
||||||
|
}
|
||||||
|
return success(truncateUtf8(raw, 80));
|
||||||
|
}
|
||||||
|
|
||||||
|
function timestampField(object: JsonObject, keys: readonly string[]): InputResult<number> {
|
||||||
|
const raw = pick(object, keys);
|
||||||
|
if (typeof raw !== "number" || !Number.isSafeInteger(raw) || raw < 0) {
|
||||||
|
return failure(400, "invalid_timestamp");
|
||||||
|
}
|
||||||
|
return success(raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
function booleanField(
|
||||||
|
object: JsonObject,
|
||||||
|
keys: readonly string[],
|
||||||
|
error: string,
|
||||||
|
required: true,
|
||||||
|
): InputResult<boolean>;
|
||||||
|
function booleanField(
|
||||||
|
object: JsonObject,
|
||||||
|
keys: readonly string[],
|
||||||
|
error: string,
|
||||||
|
required: false,
|
||||||
|
): InputResult<boolean | undefined>;
|
||||||
|
function booleanField(
|
||||||
|
object: JsonObject,
|
||||||
|
keys: readonly string[],
|
||||||
|
error: string,
|
||||||
|
required: boolean,
|
||||||
|
): InputResult<boolean | undefined> {
|
||||||
|
const raw = pick(object, keys);
|
||||||
|
if (raw === MISSING) {
|
||||||
|
return required ? failure(400, error) : success(undefined);
|
||||||
|
}
|
||||||
|
return typeof raw === "boolean" ? success(raw) : failure(400, error);
|
||||||
|
}
|
||||||
|
|
||||||
|
function schemaVersion(object: JsonObject): InputResult<1> {
|
||||||
|
const raw = pick(object, ["schemaVersion", "schema_version"]);
|
||||||
|
if (raw === MISSING || raw === 1) return success(1);
|
||||||
|
return failure(400, "unsupported_schema_version");
|
||||||
|
}
|
||||||
|
|
||||||
|
function pick(object: JsonObject, keys: readonly string[]): unknown | typeof MISSING {
|
||||||
|
for (const key of keys) {
|
||||||
|
if (Object.prototype.hasOwnProperty.call(object, key)) return object[key];
|
||||||
|
}
|
||||||
|
return MISSING;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlainObject(value: unknown): value is JsonObject {
|
||||||
|
if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
|
||||||
|
const prototype = Object.getPrototypeOf(value);
|
||||||
|
return prototype === Object.prototype || prototype === null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isApplicationJson(contentType: string | null): boolean {
|
||||||
|
if (contentType == null) return false;
|
||||||
|
const [mediaType, ...parameters] = contentType.split(";");
|
||||||
|
if (mediaType?.trim().toLowerCase() !== "application/json") return false;
|
||||||
|
for (const parameter of parameters) {
|
||||||
|
const separator = parameter.indexOf("=");
|
||||||
|
if (separator < 0) continue;
|
||||||
|
if (parameter.slice(0, separator).trim().toLowerCase() !== "charset") continue;
|
||||||
|
const charset = parameter
|
||||||
|
.slice(separator + 1)
|
||||||
|
.trim()
|
||||||
|
.replace(/^"(.*)"$/, "$1")
|
||||||
|
.toLowerCase();
|
||||||
|
if (charset !== "utf-8" && charset !== "utf8") return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function joinChunks(chunks: readonly Uint8Array[], totalBytes: number): Uint8Array {
|
||||||
|
if (chunks.length === 1) return chunks[0] ?? new Uint8Array();
|
||||||
|
const bytes = new Uint8Array(totalBytes);
|
||||||
|
let offset = 0;
|
||||||
|
for (const chunk of chunks) {
|
||||||
|
bytes.set(chunk, offset);
|
||||||
|
offset += chunk.byteLength;
|
||||||
|
}
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateUtf8(value: string, maxBytes: number): string {
|
||||||
|
const bytes = UTF8_ENCODER.encode(value);
|
||||||
|
if (bytes.byteLength <= maxBytes) return value;
|
||||||
|
for (let end = maxBytes; end >= Math.max(0, maxBytes - 3); end -= 1) {
|
||||||
|
try {
|
||||||
|
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||||
|
bytes.subarray(0, end),
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// A UTF-8 boundary is at most three bytes behind the byte cap.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function jsonBytes(value: unknown): number {
|
||||||
|
return UTF8_ENCODER.encode(JSON.stringify(value)).byteLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
function success<T>(value: T): InputResult<T> {
|
||||||
|
return { ok: true, value };
|
||||||
|
}
|
||||||
|
|
||||||
|
function failure(status: 400 | 413 | 415, error: string): InputFailure {
|
||||||
|
return { ok: false, status, error };
|
||||||
|
}
|
||||||
340
services/diagnostics-api/src/storage.ts
Normal file
340
services/diagnostics-api/src/storage.ts
Normal file
@@ -0,0 +1,340 @@
|
|||||||
|
import type {
|
||||||
|
NormalizedBugPayload,
|
||||||
|
NormalizedCrashPayload,
|
||||||
|
NormalizedEventsPayload,
|
||||||
|
} from "./input";
|
||||||
|
|
||||||
|
export type DiagnosticsEnv = Cloudflare.Env & {
|
||||||
|
INGEST_KEY?: string;
|
||||||
|
AE?: AnalyticsEngineDataset;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface StoreResult {
|
||||||
|
id: string;
|
||||||
|
duplicate: boolean;
|
||||||
|
stored: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function storeEvents(
|
||||||
|
payload: NormalizedEventsPayload,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<StoreResult> {
|
||||||
|
const result = await env.DB.prepare(
|
||||||
|
`INSERT INTO event_batches (id, received_at, install_id, app_version, platform, event_count, payload_json)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO NOTHING`,
|
||||||
|
)
|
||||||
|
.bind(
|
||||||
|
payload.batchId,
|
||||||
|
Date.now(),
|
||||||
|
payload.installId,
|
||||||
|
payload.appVersion,
|
||||||
|
payload.platform,
|
||||||
|
payload.events.length,
|
||||||
|
JSON.stringify(payload.events),
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
const duplicate = result.meta.changes === 0;
|
||||||
|
if (!duplicate && env.AE) {
|
||||||
|
try {
|
||||||
|
for (const event of payload.events) {
|
||||||
|
env.AE.writeDataPoint({
|
||||||
|
blobs: [
|
||||||
|
event.name,
|
||||||
|
payload.platform,
|
||||||
|
payload.appVersion,
|
||||||
|
payload.installId,
|
||||||
|
JSON.stringify(event.properties),
|
||||||
|
payload.batchId,
|
||||||
|
],
|
||||||
|
doubles: [event.timestampMillis, event.schemaVersion],
|
||||||
|
indexes: [payload.installId],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// D1 remains the durable source of truth if the optional analytics index is unavailable.
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "failed to index diagnostics event batch",
|
||||||
|
batchId: payload.batchId,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
id: payload.batchId,
|
||||||
|
duplicate,
|
||||||
|
stored: duplicate ? 0 : payload.events.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function storeCrash(
|
||||||
|
payload: NormalizedCrashPayload,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<StoreResult & { fingerprint: string }> {
|
||||||
|
const database = env.DB.withSession("first-primary");
|
||||||
|
const existing = await database
|
||||||
|
.prepare("SELECT fingerprint FROM crashes WHERE id = ?")
|
||||||
|
.bind(payload.id)
|
||||||
|
.first<{ fingerprint: string }>();
|
||||||
|
if (existing) {
|
||||||
|
return { id: payload.id, duplicate: true, stored: 0, fingerprint: existing.fingerprint };
|
||||||
|
}
|
||||||
|
|
||||||
|
const fingerprint = await crashFingerprint(payload.exceptionType, payload.stackTrace);
|
||||||
|
const stackKey = payload.stackTrace
|
||||||
|
? `crashes/${payload.id}/${crypto.randomUUID()}/stack.txt`
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (stackKey) {
|
||||||
|
await env.BLOBS.put(stackKey, payload.stackTrace, {
|
||||||
|
httpMetadata: { contentType: "text/plain; charset=utf-8" },
|
||||||
|
customMetadata: { installId: payload.installId, fingerprint },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await database
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO crashes (
|
||||||
|
id, received_at, occurred_at, install_id, app_version, platform,
|
||||||
|
exception_type, exception_message, fingerprint, diagnostics_enabled,
|
||||||
|
stack_r2_key, breadcrumbs_json, schema_version
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO NOTHING`,
|
||||||
|
)
|
||||||
|
.bind(
|
||||||
|
payload.id,
|
||||||
|
Date.now(),
|
||||||
|
payload.occurredAt,
|
||||||
|
payload.installId,
|
||||||
|
payload.appVersion,
|
||||||
|
payload.platform,
|
||||||
|
payload.exceptionType,
|
||||||
|
payload.exceptionMessage,
|
||||||
|
fingerprint,
|
||||||
|
payload.diagnosticsEnabledAtCapture ? 1 : 0,
|
||||||
|
stackKey,
|
||||||
|
JSON.stringify(payload.breadcrumbs),
|
||||||
|
payload.schemaVersion,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
const duplicate = result.meta.changes === 0;
|
||||||
|
if (duplicate) {
|
||||||
|
const stored = await database
|
||||||
|
.prepare("SELECT fingerprint FROM crashes WHERE id = ?")
|
||||||
|
.bind(payload.id)
|
||||||
|
.first<{ fingerprint: string }>();
|
||||||
|
if (!stored) throw new Error("duplicate crash row was not readable");
|
||||||
|
if (stackKey) await deleteAttemptBlob(env, stackKey);
|
||||||
|
return { id: payload.id, duplicate: true, stored: 0, fingerprint: stored.fingerprint };
|
||||||
|
}
|
||||||
|
return { id: payload.id, duplicate: false, stored: 1, fingerprint };
|
||||||
|
} catch (error) {
|
||||||
|
if (stackKey) {
|
||||||
|
await deleteAttemptBlob(env, stackKey);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function storeBug(
|
||||||
|
payload: NormalizedBugPayload,
|
||||||
|
env: DiagnosticsEnv,
|
||||||
|
): Promise<StoreResult> {
|
||||||
|
const database = env.DB.withSession("first-primary");
|
||||||
|
const existing = await database
|
||||||
|
.prepare("SELECT id FROM bugs WHERE id = ?")
|
||||||
|
.bind(payload.id)
|
||||||
|
.first<{ id: string }>();
|
||||||
|
if (existing) return { id: payload.id, duplicate: true, stored: 0 };
|
||||||
|
|
||||||
|
const logsKey = payload.logs ? `bugs/${payload.id}/${crypto.randomUUID()}/logs.txt` : null;
|
||||||
|
if (logsKey) {
|
||||||
|
await env.BLOBS.put(logsKey, payload.logs, {
|
||||||
|
httpMetadata: { contentType: "text/plain; charset=utf-8" },
|
||||||
|
customMetadata: { installId: payload.installId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await database
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO bugs (
|
||||||
|
id, received_at, occurred_at, install_id, app_version, platform,
|
||||||
|
what_happened, expected, steps, contact, logs_r2_key,
|
||||||
|
device_json, breadcrumbs_json, status, schema_version
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?)
|
||||||
|
ON CONFLICT(id) DO NOTHING`,
|
||||||
|
)
|
||||||
|
.bind(
|
||||||
|
payload.id,
|
||||||
|
Date.now(),
|
||||||
|
payload.occurredAt,
|
||||||
|
payload.installId,
|
||||||
|
payload.appVersion,
|
||||||
|
payload.platform,
|
||||||
|
payload.whatHappened,
|
||||||
|
payload.expected,
|
||||||
|
payload.steps,
|
||||||
|
payload.contact,
|
||||||
|
logsKey,
|
||||||
|
JSON.stringify(payload.device),
|
||||||
|
JSON.stringify(payload.breadcrumbs),
|
||||||
|
payload.schemaVersion,
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
const duplicate = result.meta.changes === 0;
|
||||||
|
if (duplicate && logsKey) {
|
||||||
|
await deleteAttemptBlob(env, logsKey);
|
||||||
|
}
|
||||||
|
return { id: payload.id, duplicate, stored: duplicate ? 0 : 1 };
|
||||||
|
} catch (error) {
|
||||||
|
if (logsKey) {
|
||||||
|
await deleteAttemptBlob(env, logsKey);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runRetention(env: DiagnosticsEnv): Promise<void> {
|
||||||
|
const retentionDays = boundedPositiveInt(env.RETENTION_DAYS, 90, 1, 3_650);
|
||||||
|
const cutoff = Date.now() - retentionDays * 86_400_000;
|
||||||
|
// Eight full passes plus the backlog check use at most 43 of D1's 50 queries per invocation.
|
||||||
|
for (let pass = 0; pass < 8; pass += 1) {
|
||||||
|
const hasFullBatch = await runRetentionPass(env, cutoff);
|
||||||
|
if (!hasFullBatch) return;
|
||||||
|
}
|
||||||
|
const [events, crashes, bugs] = await env.DB.batch<{ count: number }>([
|
||||||
|
env.DB.prepare("SELECT COUNT(*) AS count FROM event_batches WHERE received_at < ?").bind(
|
||||||
|
cutoff,
|
||||||
|
),
|
||||||
|
env.DB.prepare("SELECT COUNT(*) AS count FROM crashes WHERE received_at < ?").bind(cutoff),
|
||||||
|
env.DB.prepare("SELECT COUNT(*) AS count FROM bugs WHERE received_at < ?").bind(cutoff),
|
||||||
|
]);
|
||||||
|
console.warn(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "diagnostics retention reached its per-run pass limit",
|
||||||
|
cutoff,
|
||||||
|
backlog: {
|
||||||
|
eventBatches: events.results[0]?.count ?? 0,
|
||||||
|
crashes: crashes.results[0]?.count ?? 0,
|
||||||
|
bugs: bugs.results[0]?.count ?? 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runRetentionPass(env: DiagnosticsEnv, cutoff: number): Promise<boolean> {
|
||||||
|
const reportBatchSize = 900;
|
||||||
|
const eventBatchSize = 1_000;
|
||||||
|
const [crashes, bugs] = await Promise.all([
|
||||||
|
expiredBlobRows(env.DB, "crashes", "stack_r2_key", cutoff, reportBatchSize),
|
||||||
|
expiredBlobRows(env.DB, "bugs", "logs_r2_key", cutoff, reportBatchSize),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const blobKeys = [...crashes, ...bugs]
|
||||||
|
.map((row) => row.blobKey)
|
||||||
|
.filter((key): key is string => key !== null);
|
||||||
|
for (let offset = 0; offset < blobKeys.length; offset += 1_000) {
|
||||||
|
await env.BLOBS.delete(blobKeys.slice(offset, offset + 1_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
const statements = [retentionStatement(env.DB, "event_batches", cutoff, eventBatchSize)];
|
||||||
|
if (crashes.length > 0) statements.push(deleteRowsById(env.DB, "crashes", crashes));
|
||||||
|
if (bugs.length > 0) statements.push(deleteRowsById(env.DB, "bugs", bugs));
|
||||||
|
const [eventsResult] = await env.DB.batch(statements);
|
||||||
|
return (
|
||||||
|
eventsResult.meta.changes === eventBatchSize ||
|
||||||
|
crashes.length === reportBatchSize ||
|
||||||
|
bugs.length === reportBatchSize
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ExpiredBlobRow {
|
||||||
|
id: string;
|
||||||
|
blobKey: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function expiredBlobRows(
|
||||||
|
database: D1Database,
|
||||||
|
table: "crashes" | "bugs",
|
||||||
|
column: "stack_r2_key" | "logs_r2_key",
|
||||||
|
cutoff: number,
|
||||||
|
batchSize: number,
|
||||||
|
): Promise<ExpiredBlobRow[]> {
|
||||||
|
const result = await database
|
||||||
|
.prepare(
|
||||||
|
`SELECT id, ${column} AS blobKey
|
||||||
|
FROM ${table}
|
||||||
|
WHERE received_at < ?
|
||||||
|
ORDER BY received_at
|
||||||
|
LIMIT ?`,
|
||||||
|
)
|
||||||
|
.bind(cutoff, batchSize)
|
||||||
|
.all<ExpiredBlobRow>();
|
||||||
|
return result.results;
|
||||||
|
}
|
||||||
|
|
||||||
|
function retentionStatement(
|
||||||
|
database: D1Database,
|
||||||
|
table: "event_batches" | "crashes" | "bugs",
|
||||||
|
cutoff: number,
|
||||||
|
batchSize: number,
|
||||||
|
): D1PreparedStatement {
|
||||||
|
return database
|
||||||
|
.prepare(
|
||||||
|
`DELETE FROM ${table}
|
||||||
|
WHERE rowid IN (
|
||||||
|
SELECT rowid FROM ${table} WHERE received_at < ? ORDER BY received_at LIMIT ?
|
||||||
|
)`,
|
||||||
|
)
|
||||||
|
.bind(cutoff, batchSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
function deleteRowsById(
|
||||||
|
database: D1Database,
|
||||||
|
table: "crashes" | "bugs",
|
||||||
|
rows: ExpiredBlobRow[],
|
||||||
|
): D1PreparedStatement {
|
||||||
|
return database
|
||||||
|
.prepare(`DELETE FROM ${table} WHERE id IN (SELECT value FROM json_each(?))`)
|
||||||
|
.bind(JSON.stringify(rows.map((row) => row.id)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function crashFingerprint(exceptionType: string, stackTrace: string): Promise<string> {
|
||||||
|
const topFrames = stackTrace
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(0, 4)
|
||||||
|
.join("\n");
|
||||||
|
const bytes = new TextEncoder().encode(`${exceptionType}\n${topFrames}`);
|
||||||
|
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes));
|
||||||
|
return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteAttemptBlob(env: DiagnosticsEnv, key: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await env.BLOBS.delete(key);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
JSON.stringify({
|
||||||
|
message: "failed to remove uncommitted diagnostics blob",
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function boundedPositiveInt(
|
||||||
|
raw: string | undefined,
|
||||||
|
fallback: number,
|
||||||
|
minimum: number,
|
||||||
|
maximum: number,
|
||||||
|
): number {
|
||||||
|
const parsed = Number(raw);
|
||||||
|
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) return fallback;
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
4
services/diagnostics-api/test/apply-migrations.ts
Normal file
4
services/diagnostics-api/test/apply-migrations.ts
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
import { env } from "cloudflare:workers";
|
||||||
|
import { applyD1Migrations } from "cloudflare:test";
|
||||||
|
|
||||||
|
await applyD1Migrations(env.DB, env.TEST_MIGRATIONS);
|
||||||
12
services/diagnostics-api/test/env.d.ts
vendored
Normal file
12
services/diagnostics-api/test/env.d.ts
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
import type { D1Migration } from "@cloudflare/vitest-pool-workers";
|
||||||
|
|
||||||
|
declare global {
|
||||||
|
namespace Cloudflare {
|
||||||
|
interface Env {
|
||||||
|
INGEST_KEY: string;
|
||||||
|
TEST_MIGRATIONS: D1Migration[];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export {};
|
||||||
266
services/diagnostics-api/test/input.test.ts
Normal file
266
services/diagnostics-api/test/input.test.ts
Normal file
@@ -0,0 +1,266 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
MAX_BREADCRUMBS_JSON_BYTES,
|
||||||
|
MAX_DEVICE_JSON_BYTES,
|
||||||
|
MAX_LOG_BYTES,
|
||||||
|
normalizeBug,
|
||||||
|
normalizeCrash,
|
||||||
|
normalizeEvents,
|
||||||
|
readJsonObject,
|
||||||
|
} from "../src/input";
|
||||||
|
|
||||||
|
const ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||||
|
const INSTALL_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
|
||||||
|
const ENCODER = new TextEncoder();
|
||||||
|
|
||||||
|
describe("readJsonObject", () => {
|
||||||
|
it("enforces the byte limit on a chunked body without Content-Length", async () => {
|
||||||
|
const bytes = ENCODER.encode(JSON.stringify({ value: "😀".repeat(40) }));
|
||||||
|
const request = chunkedJsonRequest([
|
||||||
|
bytes.subarray(0, 7),
|
||||||
|
bytes.subarray(7, 31),
|
||||||
|
bytes.subarray(31),
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(request.headers.has("content-length")).toBe(false);
|
||||||
|
await expect(readJsonObject(request, 64)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 413,
|
||||||
|
error: "payload_too_large",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("joins chunks before fatally decoding UTF-8", async () => {
|
||||||
|
const bytes = ENCODER.encode(JSON.stringify({ value: "😀" }));
|
||||||
|
const emojiStart = bytes.indexOf(0xf0);
|
||||||
|
const request = chunkedJsonRequest([
|
||||||
|
bytes.subarray(0, emojiStart + 2),
|
||||||
|
bytes.subarray(emojiStart + 2),
|
||||||
|
]);
|
||||||
|
|
||||||
|
await expect(readJsonObject(request, bytes.byteLength)).resolves.toEqual({
|
||||||
|
ok: true,
|
||||||
|
value: { value: "😀" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects malformed UTF-8", async () => {
|
||||||
|
const request = chunkedJsonRequest([
|
||||||
|
new Uint8Array([0x7b, 0x22, 0x78, 0x22, 0x3a, 0x22, 0xc3, 0x28, 0x22, 0x7d]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
await expect(readJsonObject(request, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 400,
|
||||||
|
error: "invalid_utf8",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires application/json with a UTF-8 charset", async () => {
|
||||||
|
const missing = new Request("https://example.test/v1/events", {
|
||||||
|
method: "POST",
|
||||||
|
body: "{}",
|
||||||
|
});
|
||||||
|
const wrongCharset = chunkedJsonRequest([ENCODER.encode("{}")], "application/json; charset=utf-16");
|
||||||
|
|
||||||
|
await expect(readJsonObject(missing, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 415,
|
||||||
|
error: "unsupported_media_type",
|
||||||
|
});
|
||||||
|
await expect(readJsonObject(wrongCharset, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 415,
|
||||||
|
error: "unsupported_media_type",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("validates Content-Length before reading the body", async () => {
|
||||||
|
const invalid = chunkedJsonRequest([ENCODER.encode("{}")], "application/json", "invalid");
|
||||||
|
const oversized = chunkedJsonRequest(
|
||||||
|
[ENCODER.encode("{}")],
|
||||||
|
"application/json",
|
||||||
|
"999999999999999999999999999999999999",
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(readJsonObject(invalid, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 400,
|
||||||
|
error: "invalid_content_length",
|
||||||
|
});
|
||||||
|
await expect(readJsonObject(oversized, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 413,
|
||||||
|
error: "payload_too_large",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a non-object JSON root", async () => {
|
||||||
|
const request = chunkedJsonRequest([ENCODER.encode("null")]);
|
||||||
|
|
||||||
|
await expect(readJsonObject(request, 100)).resolves.toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 400,
|
||||||
|
error: "invalid_body",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("normalizers", () => {
|
||||||
|
it("preserves false booleans and rejects their string representation", () => {
|
||||||
|
const crash = crashPayload(false);
|
||||||
|
const normalizedCrash = normalizeCrash(crash);
|
||||||
|
expect(normalizedCrash.ok).toBe(true);
|
||||||
|
if (normalizedCrash.ok) {
|
||||||
|
expect(normalizedCrash.value.diagnosticsEnabledAtCapture).toBe(false);
|
||||||
|
}
|
||||||
|
expect(normalizeCrash(crashPayload("false"))).toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 400,
|
||||||
|
error: "invalid_diagnostics_enabled",
|
||||||
|
});
|
||||||
|
|
||||||
|
const bug = bugPayload({ include_logs: false, logs: "discard me" });
|
||||||
|
const normalizedBug = normalizeBug(bug);
|
||||||
|
expect(normalizedBug.ok).toBe(true);
|
||||||
|
if (normalizedBug.ok) expect(normalizedBug.value.logs).toBe("");
|
||||||
|
const missingConsent = normalizeBug(bugPayload({ logs: "discard me" }));
|
||||||
|
expect(missingConsent.ok && missingConsent.value.logs).toBe("");
|
||||||
|
expect(normalizeBug(bugPayload({ include_logs: "false" }))).toEqual({
|
||||||
|
ok: false,
|
||||||
|
status: 400,
|
||||||
|
error: "invalid_include_logs",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps logs, breadcrumbs, and device JSON within valid byte budgets", () => {
|
||||||
|
const properties = Object.fromEntries(
|
||||||
|
Array.from({ length: 12 }, (_, index) => [`key-${index}-${"\u0000".repeat(40)}`, "\u0000".repeat(128)]),
|
||||||
|
);
|
||||||
|
const breadcrumbs = Array.from({ length: 40 }, (_, index) => ({
|
||||||
|
name: `crumb-${index}`,
|
||||||
|
timestamp_millis: index,
|
||||||
|
properties,
|
||||||
|
}));
|
||||||
|
const result = normalizeBug(
|
||||||
|
bugPayload({
|
||||||
|
include_logs: true,
|
||||||
|
logs: "😀".repeat(60_000),
|
||||||
|
breadcrumbs,
|
||||||
|
device: {
|
||||||
|
device_name: "\u0000".repeat(200),
|
||||||
|
device_model: "\u0000".repeat(200),
|
||||||
|
operating_system: "\u0000".repeat(300),
|
||||||
|
network: "\u0000".repeat(150),
|
||||||
|
battery_level: "\u0000".repeat(100),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
if (!result.ok) return;
|
||||||
|
const breadcrumbsJson = JSON.stringify(result.value.breadcrumbs);
|
||||||
|
const deviceJson = JSON.stringify(result.value.device);
|
||||||
|
expect(ENCODER.encode(result.value.logs).byteLength).toBe(MAX_LOG_BYTES);
|
||||||
|
expect(ENCODER.encode(breadcrumbsJson).byteLength).toBeLessThanOrEqual(
|
||||||
|
MAX_BREADCRUMBS_JSON_BYTES,
|
||||||
|
);
|
||||||
|
expect(ENCODER.encode(deviceJson).byteLength).toBeLessThanOrEqual(MAX_DEVICE_JSON_BYTES);
|
||||||
|
expect(JSON.parse(breadcrumbsJson)).toEqual(result.value.breadcrumbs);
|
||||||
|
expect(JSON.parse(deviceJson)).toEqual(result.value.device);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires stable report IDs and validates supplied IDs and schema versions", () => {
|
||||||
|
const result = normalizeEvents({
|
||||||
|
events: [{ name: "opened", ts: 1, schema_version: 1 }],
|
||||||
|
});
|
||||||
|
expect(result).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
|
||||||
|
const legacyInstall = normalizeEvents({
|
||||||
|
batch_id: ID,
|
||||||
|
install_id: "legacy-test-install",
|
||||||
|
events: [{ name: "opened", ts: 1 }],
|
||||||
|
});
|
||||||
|
expect(legacyInstall.ok && legacyInstall.value.installId).toBe("legacy-test-install");
|
||||||
|
const missingInstall = normalizeEvents({
|
||||||
|
batch_id: ID,
|
||||||
|
events: [{ name: "opened", ts: 1 }],
|
||||||
|
});
|
||||||
|
expect(missingInstall.ok && missingInstall.value.installId).toBe("unknown");
|
||||||
|
expect(
|
||||||
|
normalizeEvents({
|
||||||
|
batch_id: ID,
|
||||||
|
install_id: "bad\u0000install",
|
||||||
|
events: [{ name: "opened", ts: 1 }],
|
||||||
|
}),
|
||||||
|
).toEqual({ ok: false, status: 400, error: "invalid_install_id" });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
normalizeEvents({
|
||||||
|
batch_id: "not-a-uuid",
|
||||||
|
events: [{ name: "opened", timestamp_millis: 1 }],
|
||||||
|
}),
|
||||||
|
).toEqual({ ok: false, status: 400, error: "invalid_batch_id" });
|
||||||
|
expect(
|
||||||
|
normalizeEvents({
|
||||||
|
batch_id: ID,
|
||||||
|
install_id: INSTALL_ID,
|
||||||
|
events: [{ name: "opened", timestamp_millis: 1, schema_version: 2 }],
|
||||||
|
}),
|
||||||
|
).toEqual({ ok: false, status: 400, error: "unsupported_schema_version" });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function chunkedJsonRequest(
|
||||||
|
chunks: readonly Uint8Array[],
|
||||||
|
contentType = "application/json; charset=utf-8",
|
||||||
|
contentLength?: string,
|
||||||
|
): Request {
|
||||||
|
return new Request("https://example.test/v1/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"content-type": contentType,
|
||||||
|
...(contentLength == null ? {} : { "content-length": contentLength }),
|
||||||
|
},
|
||||||
|
body: new ReadableStream<Uint8Array>({
|
||||||
|
start(controller) {
|
||||||
|
for (const chunk of chunks) controller.enqueue(chunk);
|
||||||
|
controller.close();
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function crashPayload(diagnosticsEnabled: unknown): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id: ID,
|
||||||
|
install_id: INSTALL_ID,
|
||||||
|
app_version: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
exception_type: "ExampleError",
|
||||||
|
exception_message: "message",
|
||||||
|
stack_trace: "stack",
|
||||||
|
occurred_at: 1,
|
||||||
|
diagnostics_enabled: diagnosticsEnabled,
|
||||||
|
schema_version: 1,
|
||||||
|
breadcrumbs: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function bugPayload(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id: ID,
|
||||||
|
install_id: INSTALL_ID,
|
||||||
|
app_version: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
occurred_at: 1,
|
||||||
|
what_happened: "It failed",
|
||||||
|
expected: "It worked",
|
||||||
|
steps: "Open the app",
|
||||||
|
contact: "",
|
||||||
|
logs: "",
|
||||||
|
device: {},
|
||||||
|
breadcrumbs: [],
|
||||||
|
schema_version: 1,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
7
services/diagnostics-api/test/tsconfig.json
Normal file
7
services/diagnostics-api/test/tsconfig.json
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"extends": "../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"types": ["@cloudflare/vitest-pool-workers/types", "node"]
|
||||||
|
},
|
||||||
|
"include": ["../worker-configuration.d.ts", "../vitest.config.ts", "./**/*.ts"]
|
||||||
|
}
|
||||||
606
services/diagnostics-api/test/worker.test.ts
Normal file
606
services/diagnostics-api/test/worker.test.ts
Normal file
@@ -0,0 +1,606 @@
|
|||||||
|
import { env, exports } from "cloudflare:workers";
|
||||||
|
import { createExecutionContext } from "cloudflare:test";
|
||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import worker from "../src/index";
|
||||||
|
import type {
|
||||||
|
NormalizedBugPayload,
|
||||||
|
NormalizedCrashPayload,
|
||||||
|
NormalizedEventsPayload,
|
||||||
|
} from "../src/input";
|
||||||
|
import {
|
||||||
|
type DiagnosticsEnv,
|
||||||
|
runRetention,
|
||||||
|
storeBug,
|
||||||
|
storeCrash,
|
||||||
|
storeEvents,
|
||||||
|
} from "../src/storage";
|
||||||
|
|
||||||
|
const INSTALL_ID = "10000000-0000-4000-8000-000000000000";
|
||||||
|
|
||||||
|
describe("diagnostics Worker", () => {
|
||||||
|
it("keeps public routing narrow and fails closed", async () => {
|
||||||
|
const live = await exports.default.fetch(new Request("https://diagnostics.test/live"));
|
||||||
|
expect(live.status).toBe(200);
|
||||||
|
expect(await live.json()).toMatchObject({ ok: true, schema: 1 });
|
||||||
|
|
||||||
|
const health = await exports.default.fetch(healthRequest());
|
||||||
|
expect(health.status).toBe(200);
|
||||||
|
expect(await health.json()).toMatchObject({ ok: true });
|
||||||
|
const unauthorizedHealth = await exports.default.fetch(healthRequest("wrong-key"));
|
||||||
|
expect(unauthorizedHealth.status).toBe(401);
|
||||||
|
|
||||||
|
const unknown = await exports.default.fetch(
|
||||||
|
new Request("https://diagnostics.test/v1/not-real", { method: "POST" }),
|
||||||
|
);
|
||||||
|
expect(unknown.status).toBe(404);
|
||||||
|
|
||||||
|
const unauthorized = await exports.default.fetch(
|
||||||
|
jsonRequest("/v1/events", eventPayload(uuid(1)), "wrong-key"),
|
||||||
|
);
|
||||||
|
expect(unauthorized.status).toBe(401);
|
||||||
|
expect(await unauthorized.json()).toEqual({ error: "unauthorized" });
|
||||||
|
expect(unauthorized.headers.get("x-request-id")).toMatch(
|
||||||
|
/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/,
|
||||||
|
);
|
||||||
|
|
||||||
|
const preflight = await exports.default.fetch(
|
||||||
|
new Request("https://diagnostics.test/v1/events", { method: "OPTIONS" }),
|
||||||
|
);
|
||||||
|
expect(preflight.status).toBe(204);
|
||||||
|
expect(preflight.headers.get("access-control-allow-origin")).toBeNull();
|
||||||
|
expect(preflight.headers.get("allow")).toBe("POST, OPTIONS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies the source limit before shared-key verification", async () => {
|
||||||
|
let installLimitCalls = 0;
|
||||||
|
const limitedEnv: DiagnosticsEnv = {
|
||||||
|
...env,
|
||||||
|
SOURCE_RATE_LIMITER: {
|
||||||
|
limit: async () => ({ success: false }),
|
||||||
|
} as RateLimit,
|
||||||
|
INSTALL_RATE_LIMITER: {
|
||||||
|
limit: async () => {
|
||||||
|
installLimitCalls += 1;
|
||||||
|
return { success: true };
|
||||||
|
},
|
||||||
|
} as RateLimit,
|
||||||
|
};
|
||||||
|
const context = createExecutionContext();
|
||||||
|
|
||||||
|
const response = await worker.fetch(
|
||||||
|
jsonRequest("/v1/events", eventPayload(uuid(3)), "wrong-key", "198.51.100.3"),
|
||||||
|
limitedEnv,
|
||||||
|
context,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(429);
|
||||||
|
expect(response.headers.get("retry-after")).toBe("60");
|
||||||
|
expect(await response.json()).toEqual({ error: "rate_limited" });
|
||||||
|
expect(installLimitCalls).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns structured errors for invalid bodies and asynchronous storage failures", async () => {
|
||||||
|
const invalid = await exports.default.fetch(jsonRequest("/v1/events", null));
|
||||||
|
expect(invalid.status).toBe(400);
|
||||||
|
expect(await invalid.json()).toEqual({ error: "invalid_body" });
|
||||||
|
|
||||||
|
const rejection = new Error("simulated D1 rejection");
|
||||||
|
const statement = {
|
||||||
|
bind: () => statement,
|
||||||
|
first: async () => Promise.reject(rejection),
|
||||||
|
run: async () => Promise.reject(rejection),
|
||||||
|
};
|
||||||
|
const rejectingDatabase = {
|
||||||
|
prepare: () => statement,
|
||||||
|
} as unknown as D1Database;
|
||||||
|
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
|
||||||
|
|
||||||
|
const healthContext = createExecutionContext();
|
||||||
|
const unhealthy = await worker.fetch(
|
||||||
|
healthRequest(env.INGEST_KEY, "198.51.100.4"),
|
||||||
|
rejectingEnv,
|
||||||
|
healthContext,
|
||||||
|
);
|
||||||
|
expect(unhealthy.status).toBe(503);
|
||||||
|
expect(await unhealthy.json()).toEqual({ ok: false, error: "dependency_unavailable" });
|
||||||
|
|
||||||
|
const ingestContext = createExecutionContext();
|
||||||
|
const failedIngest = await worker.fetch(
|
||||||
|
jsonRequest("/v1/events", eventPayload(uuid(2)), env.INGEST_KEY, "198.51.100.2"),
|
||||||
|
rejectingEnv,
|
||||||
|
ingestContext,
|
||||||
|
);
|
||||||
|
expect(failedIngest.status).toBe(500);
|
||||||
|
expect(await failedIngest.json()).toEqual({ error: "internal" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates event batches using the client batch ID", async () => {
|
||||||
|
const id = uuid(10);
|
||||||
|
const first = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
|
||||||
|
const second = await exports.default.fetch(jsonRequest("/v1/events", eventPayload(id)));
|
||||||
|
|
||||||
|
expect(first.status).toBe(202);
|
||||||
|
expect(await first.json()).toMatchObject({
|
||||||
|
ok: true,
|
||||||
|
id,
|
||||||
|
stored: 1,
|
||||||
|
duplicate: false,
|
||||||
|
});
|
||||||
|
expect(second.status).toBe(202);
|
||||||
|
expect(await second.json()).toMatchObject({
|
||||||
|
ok: true,
|
||||||
|
id,
|
||||||
|
stored: 0,
|
||||||
|
duplicate: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const row = await env.DB.prepare(
|
||||||
|
"SELECT event_count AS eventCount, payload_json AS payloadJson FROM event_batches WHERE id = ?",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.first<{ eventCount: number; payloadJson: string }>();
|
||||||
|
expect(row?.eventCount).toBe(1);
|
||||||
|
expect(JSON.parse(row?.payloadJson ?? "null")).toEqual([
|
||||||
|
{
|
||||||
|
name: "app_open",
|
||||||
|
timestampMillis: 1,
|
||||||
|
properties: { screen: "home" },
|
||||||
|
schemaVersion: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps D1 idempotency when the optional analytics index is enabled", async () => {
|
||||||
|
const points: AnalyticsEngineDataPoint[] = [];
|
||||||
|
const analytics = {
|
||||||
|
writeDataPoint: (point: AnalyticsEngineDataPoint) => points.push(point),
|
||||||
|
} as AnalyticsEngineDataset;
|
||||||
|
const analyticsEnv: DiagnosticsEnv = { ...env, AE: analytics };
|
||||||
|
const payload: NormalizedEventsPayload = {
|
||||||
|
batchId: uuid(11),
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
events: [
|
||||||
|
{
|
||||||
|
name: "indexed",
|
||||||
|
timestampMillis: 1,
|
||||||
|
properties: {},
|
||||||
|
schemaVersion: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: false, stored: 1 });
|
||||||
|
expect(await storeEvents(payload, analyticsEnv)).toMatchObject({ duplicate: true, stored: 0 });
|
||||||
|
expect(points).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the accepted crash blob when a duplicate request arrives", async () => {
|
||||||
|
const id = uuid(20);
|
||||||
|
const first = await exports.default.fetch(
|
||||||
|
jsonRequest("/v1/crashes", crashPayload(id, "first stack")),
|
||||||
|
);
|
||||||
|
const second = await exports.default.fetch(
|
||||||
|
jsonRequest("/v1/crashes", crashPayload(id, "second stack")),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(first.status).toBe(202);
|
||||||
|
const firstBody = await first.json<{ fingerprint: string }>();
|
||||||
|
expect(firstBody).toMatchObject({ ok: true, id, duplicate: false });
|
||||||
|
expect(second.status).toBe(202);
|
||||||
|
const secondBody = await second.json<{ fingerprint: string }>();
|
||||||
|
expect(secondBody).toMatchObject({ ok: true, id, duplicate: true });
|
||||||
|
|
||||||
|
const row = await env.DB.prepare(
|
||||||
|
`SELECT stack_r2_key AS stackKey, breadcrumbs_json AS breadcrumbsJson,
|
||||||
|
fingerprint
|
||||||
|
FROM crashes WHERE id = ?`,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.first<{ stackKey: string; breadcrumbsJson: string; fingerprint: string }>();
|
||||||
|
expect(row?.stackKey).toMatch(new RegExp(`^crashes/${id}/[0-9a-f-]+/stack\\.txt$`));
|
||||||
|
expect(firstBody.fingerprint).toBe(row?.fingerprint);
|
||||||
|
expect(secondBody.fingerprint).toBe(row?.fingerprint);
|
||||||
|
expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([]);
|
||||||
|
expect(await (await env.BLOBS.get(row?.stackKey ?? "missing"))?.text()).toBe("first stack");
|
||||||
|
|
||||||
|
const objects = await env.BLOBS.list({ prefix: `crashes/${id}/` });
|
||||||
|
expect(objects.objects.map((object) => object.key)).toEqual([row?.stackKey]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores bug metadata as JSON and cleans the duplicate upload attempt", async () => {
|
||||||
|
const id = uuid(30);
|
||||||
|
const payload = bugPayload(id, "first logs");
|
||||||
|
const first = await exports.default.fetch(jsonRequest("/v1/bugs", payload));
|
||||||
|
const second = await exports.default.fetch(
|
||||||
|
jsonRequest("/v1/bugs", bugPayload(id, "second logs")),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(first.status).toBe(202);
|
||||||
|
expect(await first.json()).toMatchObject({ ok: true, id, duplicate: false });
|
||||||
|
expect(second.status).toBe(202);
|
||||||
|
expect(await second.json()).toMatchObject({ ok: true, id, duplicate: true });
|
||||||
|
|
||||||
|
const row = await env.DB.prepare(
|
||||||
|
`SELECT occurred_at AS occurredAt, logs_r2_key AS logsKey,
|
||||||
|
device_json AS deviceJson, breadcrumbs_json AS breadcrumbsJson
|
||||||
|
FROM bugs WHERE id = ?`,
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.first<{
|
||||||
|
occurredAt: number;
|
||||||
|
logsKey: string;
|
||||||
|
deviceJson: string;
|
||||||
|
breadcrumbsJson: string;
|
||||||
|
}>();
|
||||||
|
expect(row?.occurredAt).toBe(3);
|
||||||
|
expect(JSON.parse(row?.deviceJson ?? "null")).toEqual({
|
||||||
|
deviceName: "Test device",
|
||||||
|
deviceModel: "Model",
|
||||||
|
operatingSystem: "Test OS",
|
||||||
|
network: "offline",
|
||||||
|
batteryLevel: "90%",
|
||||||
|
});
|
||||||
|
expect(JSON.parse(row?.breadcrumbsJson ?? "null")).toEqual([
|
||||||
|
{ name: "opened", timestampMillis: 2, properties: {} },
|
||||||
|
]);
|
||||||
|
expect(await (await env.BLOBS.get(row?.logsKey ?? "missing"))?.text()).toBe("first logs");
|
||||||
|
|
||||||
|
const objects = await env.BLOBS.list({ prefix: `bugs/${id}/` });
|
||||||
|
expect(objects.objects.map((object) => object.key)).toEqual([row?.logsKey]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("acknowledges known report IDs without touching an unavailable blob store", async () => {
|
||||||
|
const crash = normalizedCrash(uuid(31), "accepted stack");
|
||||||
|
const bug = normalizedBug(uuid(32), "accepted logs");
|
||||||
|
const firstCrash = await storeCrash(crash, env);
|
||||||
|
await storeBug(bug, env);
|
||||||
|
let blobWrites = 0;
|
||||||
|
const unavailableBlobs = {
|
||||||
|
put: async () => {
|
||||||
|
blobWrites += 1;
|
||||||
|
throw new Error("simulated R2 outage");
|
||||||
|
},
|
||||||
|
} as unknown as R2Bucket;
|
||||||
|
const unavailableEnv: DiagnosticsEnv = { ...env, BLOBS: unavailableBlobs };
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
storeCrash({ ...crash, stackTrace: "retry stack" }, unavailableEnv),
|
||||||
|
).resolves.toEqual({
|
||||||
|
id: crash.id,
|
||||||
|
duplicate: true,
|
||||||
|
stored: 0,
|
||||||
|
fingerprint: firstCrash.fingerprint,
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
storeBug({ ...bug, logs: "retry logs" }, unavailableEnv),
|
||||||
|
).resolves.toEqual({ id: bug.id, duplicate: true, stored: 0 });
|
||||||
|
expect(blobWrites).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes uploaded report blobs when D1 rejects the metadata write", async () => {
|
||||||
|
const rejection = new Error("simulated D1 write rejection");
|
||||||
|
const rejectingDatabase = databaseWithSession(
|
||||||
|
() => null,
|
||||||
|
async () => Promise.reject(rejection),
|
||||||
|
);
|
||||||
|
const rejectingEnv: DiagnosticsEnv = { ...env, DB: rejectingDatabase };
|
||||||
|
const crashId = uuid(33);
|
||||||
|
const bugId = uuid(34);
|
||||||
|
|
||||||
|
const crashResponse = await worker.fetch(
|
||||||
|
jsonRequest("/v1/crashes", crashPayload(crashId, "orphan candidate"), env.INGEST_KEY, "198.51.100.33"),
|
||||||
|
rejectingEnv,
|
||||||
|
createExecutionContext(),
|
||||||
|
);
|
||||||
|
const bugResponse = await worker.fetch(
|
||||||
|
jsonRequest("/v1/bugs", bugPayload(bugId, "orphan candidate"), env.INGEST_KEY, "198.51.100.34"),
|
||||||
|
rejectingEnv,
|
||||||
|
createExecutionContext(),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(crashResponse.status).toBe(500);
|
||||||
|
expect(await crashResponse.json()).toEqual({ error: "internal" });
|
||||||
|
expect(bugResponse.status).toBe(500);
|
||||||
|
expect(await bugResponse.json()).toEqual({ error: "internal" });
|
||||||
|
expect((await env.BLOBS.list({ prefix: `crashes/${crashId}/` })).objects).toEqual([]);
|
||||||
|
expect((await env.BLOBS.list({ prefix: `bugs/${bugId}/` })).objects).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes expired rows and their exact R2 objects while preserving current data", async () => {
|
||||||
|
const oldEventId = uuid(40);
|
||||||
|
const oldCrashId = uuid(41);
|
||||||
|
const oldBugId = uuid(42);
|
||||||
|
const currentEventId = uuid(43);
|
||||||
|
const oldCrashKey = `crashes/${oldCrashId}/retention/stack.txt`;
|
||||||
|
const oldBugKey = `bugs/${oldBugId}/retention/logs.txt`;
|
||||||
|
const oldReceivedAt = Date.now() - 100 * 86_400_000;
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
env.BLOBS.put(oldCrashKey, "expired crash"),
|
||||||
|
env.BLOBS.put(oldBugKey, "expired logs"),
|
||||||
|
]);
|
||||||
|
await env.DB.batch([
|
||||||
|
env.DB.prepare(
|
||||||
|
`INSERT INTO event_batches
|
||||||
|
(id, received_at, install_id, app_version, platform, event_count, payload_json)
|
||||||
|
VALUES (?, ?, ?, '', '', 1, '[]')`,
|
||||||
|
).bind(oldEventId, oldReceivedAt, INSTALL_ID),
|
||||||
|
env.DB.prepare(
|
||||||
|
`INSERT INTO event_batches
|
||||||
|
(id, received_at, install_id, app_version, platform, event_count, payload_json)
|
||||||
|
VALUES (?, ?, ?, '', '', 1, '[]')`,
|
||||||
|
).bind(currentEventId, Date.now(), INSTALL_ID),
|
||||||
|
env.DB.prepare(
|
||||||
|
`INSERT INTO crashes
|
||||||
|
(id, received_at, occurred_at, install_id, app_version, platform,
|
||||||
|
exception_type, exception_message, fingerprint, diagnostics_enabled,
|
||||||
|
stack_r2_key, breadcrumbs_json, schema_version)
|
||||||
|
VALUES (?, ?, ?, ?, '', '', 'Error', '', 'fingerprint', 1, ?, '[]', 1)`,
|
||||||
|
).bind(oldCrashId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldCrashKey),
|
||||||
|
env.DB.prepare(
|
||||||
|
`INSERT INTO bugs
|
||||||
|
(id, received_at, occurred_at, install_id, app_version, platform,
|
||||||
|
what_happened, expected, steps, contact, logs_r2_key,
|
||||||
|
device_json, breadcrumbs_json, status, schema_version)
|
||||||
|
VALUES (?, ?, ?, ?, '', '', 'failed', 'worked', '', '', ?, '{}', '[]', 'open', 1)`,
|
||||||
|
).bind(oldBugId, oldReceivedAt, oldReceivedAt, INSTALL_ID, oldBugKey),
|
||||||
|
]);
|
||||||
|
|
||||||
|
await runRetention(env);
|
||||||
|
|
||||||
|
for (const [table, id] of [
|
||||||
|
["event_batches", oldEventId],
|
||||||
|
["crashes", oldCrashId],
|
||||||
|
["bugs", oldBugId],
|
||||||
|
] as const) {
|
||||||
|
const row = await env.DB.prepare(`SELECT id FROM ${table} WHERE id = ?`).bind(id).first();
|
||||||
|
expect(row).toBeNull();
|
||||||
|
}
|
||||||
|
expect(await env.BLOBS.head(oldCrashKey)).toBeNull();
|
||||||
|
expect(await env.BLOBS.head(oldBugKey)).toBeNull();
|
||||||
|
expect(
|
||||||
|
await env.DB.prepare("SELECT id FROM event_batches WHERE id = ?").bind(currentEventId).first(),
|
||||||
|
).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds a full retention run below the D1 per-invocation query limit", async () => {
|
||||||
|
let queryCount = 0;
|
||||||
|
let batchCalls = 0;
|
||||||
|
const blobDeleteBatchSizes: number[] = [];
|
||||||
|
const rows = Array.from({ length: 900 }, (_, index) => ({
|
||||||
|
id: `expired-${index}`,
|
||||||
|
blobKey: `expired/${index}`,
|
||||||
|
}));
|
||||||
|
const database = {
|
||||||
|
prepare: () => {
|
||||||
|
const statement = {
|
||||||
|
bind: () => statement,
|
||||||
|
all: async () => {
|
||||||
|
queryCount += 1;
|
||||||
|
return d1Result(rows, 0);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return statement;
|
||||||
|
},
|
||||||
|
batch: async (statements: D1PreparedStatement[]) => {
|
||||||
|
batchCalls += 1;
|
||||||
|
queryCount += statements.length;
|
||||||
|
if (batchCalls === 9) {
|
||||||
|
return statements.map(() => d1Result([{ count: 1 }], 0));
|
||||||
|
}
|
||||||
|
return statements.map((_, index) => d1Result([], index === 0 ? 1_000 : 900));
|
||||||
|
},
|
||||||
|
} as unknown as D1Database;
|
||||||
|
const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||||
|
const blobs = {
|
||||||
|
delete: async (keys: string | string[]) => {
|
||||||
|
blobDeleteBatchSizes.push(typeof keys === "string" ? 1 : keys.length);
|
||||||
|
},
|
||||||
|
} as unknown as R2Bucket;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runRetention({ ...env, DB: database, BLOBS: blobs });
|
||||||
|
} finally {
|
||||||
|
warning.mockRestore();
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(queryCount).toBe(43);
|
||||||
|
expect(blobDeleteBatchSizes).toHaveLength(16);
|
||||||
|
expect(Math.max(...blobDeleteBatchSizes)).toBe(1_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("converges an expired report backlog across bounded retention runs", async () => {
|
||||||
|
const oldReceivedAt = Date.now() - 100 * 86_400_000;
|
||||||
|
await env.DB.prepare(
|
||||||
|
`WITH digits(value) AS (
|
||||||
|
VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9)
|
||||||
|
), sequence(value) AS (
|
||||||
|
SELECT thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value + 1
|
||||||
|
FROM digits AS thousands
|
||||||
|
CROSS JOIN digits AS hundreds
|
||||||
|
CROSS JOIN digits AS tens
|
||||||
|
CROSS JOIN digits AS ones
|
||||||
|
WHERE thousands.value * 1000 + hundreds.value * 100 + tens.value * 10 + ones.value < 7201
|
||||||
|
)
|
||||||
|
INSERT INTO crashes (
|
||||||
|
id, received_at, occurred_at, install_id, app_version, platform,
|
||||||
|
exception_type, exception_message, fingerprint, diagnostics_enabled,
|
||||||
|
stack_r2_key, breadcrumbs_json, schema_version
|
||||||
|
)
|
||||||
|
SELECT 'retention-backlog-' || printf('%04d', value), ?, ?, ?, '', '',
|
||||||
|
'Error', '', 'fingerprint-' || value, 0, NULL, '[]', 1
|
||||||
|
FROM sequence`,
|
||||||
|
)
|
||||||
|
.bind(oldReceivedAt, oldReceivedAt, INSTALL_ID)
|
||||||
|
.run();
|
||||||
|
const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runRetention(env);
|
||||||
|
const afterFirstRun = await env.DB.prepare(
|
||||||
|
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
|
||||||
|
).first<{ count: number }>();
|
||||||
|
expect(afterFirstRun?.count).toBe(1);
|
||||||
|
|
||||||
|
await runRetention(env);
|
||||||
|
const afterSecondRun = await env.DB.prepare(
|
||||||
|
"SELECT COUNT(*) AS count FROM crashes WHERE id LIKE 'retention-backlog-%'",
|
||||||
|
).first<{ count: number }>();
|
||||||
|
expect(afterSecondRun?.count).toBe(0);
|
||||||
|
} finally {
|
||||||
|
warning.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function jsonRequest(
|
||||||
|
path: string,
|
||||||
|
body: unknown,
|
||||||
|
key = env.INGEST_KEY,
|
||||||
|
source = "198.51.100.1",
|
||||||
|
): Request {
|
||||||
|
return new Request(`https://diagnostics.test${path}`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json; charset=utf-8",
|
||||||
|
"cf-connecting-ip": source,
|
||||||
|
"x-vnidrop-install-id": INSTALL_ID,
|
||||||
|
"x-vnidrop-key": key,
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function healthRequest(key = env.INGEST_KEY, source = "198.51.100.1"): Request {
|
||||||
|
return new Request("https://diagnostics.test/health", {
|
||||||
|
headers: {
|
||||||
|
"cf-connecting-ip": source,
|
||||||
|
"x-vnidrop-key": key,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function eventPayload(batchId: string): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
batchId,
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
events: [
|
||||||
|
{
|
||||||
|
name: "app_open",
|
||||||
|
timestampMillis: 1,
|
||||||
|
properties: { screen: "home" },
|
||||||
|
schemaVersion: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function crashPayload(id: string, stackTrace: string): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
exceptionType: "TestError",
|
||||||
|
exceptionMessage: "failed",
|
||||||
|
stackTrace,
|
||||||
|
timestampMillis: 2,
|
||||||
|
diagnosticsEnabledAtCapture: true,
|
||||||
|
breadcrumbs: [],
|
||||||
|
schemaVersion: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function bugPayload(id: string, logs: string): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
timestampMillis: 3,
|
||||||
|
whatHappened: "It failed",
|
||||||
|
expected: "It should work",
|
||||||
|
steps: "Open the app",
|
||||||
|
contact: "",
|
||||||
|
includeLogs: true,
|
||||||
|
logs,
|
||||||
|
device: {
|
||||||
|
deviceName: "Test device",
|
||||||
|
deviceModel: "Model",
|
||||||
|
operatingSystem: "Test OS",
|
||||||
|
network: "offline",
|
||||||
|
batteryLevel: "90%",
|
||||||
|
},
|
||||||
|
breadcrumbs: [{ name: "opened", timestampMillis: 2, properties: {} }],
|
||||||
|
schemaVersion: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizedCrash(id: string, stackTrace: string): NormalizedCrashPayload {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
exceptionType: "TestError",
|
||||||
|
exceptionMessage: "failed",
|
||||||
|
stackTrace,
|
||||||
|
occurredAt: 2,
|
||||||
|
diagnosticsEnabledAtCapture: true,
|
||||||
|
breadcrumbs: [],
|
||||||
|
schemaVersion: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizedBug(id: string, logs: string): NormalizedBugPayload {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
installId: INSTALL_ID,
|
||||||
|
appVersion: "1.0",
|
||||||
|
platform: "test",
|
||||||
|
occurredAt: 3,
|
||||||
|
whatHappened: "It failed",
|
||||||
|
expected: "It should work",
|
||||||
|
steps: "Open the app",
|
||||||
|
contact: "",
|
||||||
|
logs,
|
||||||
|
device: {
|
||||||
|
deviceName: "Test device",
|
||||||
|
deviceModel: "Model",
|
||||||
|
operatingSystem: "Test OS",
|
||||||
|
network: "offline",
|
||||||
|
batteryLevel: "90%",
|
||||||
|
},
|
||||||
|
breadcrumbs: [],
|
||||||
|
schemaVersion: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function databaseWithSession(
|
||||||
|
first: () => unknown,
|
||||||
|
run: () => Promise<D1Result>,
|
||||||
|
): D1Database {
|
||||||
|
const statement = {
|
||||||
|
bind: () => statement,
|
||||||
|
first: async () => first(),
|
||||||
|
run,
|
||||||
|
} as unknown as D1PreparedStatement;
|
||||||
|
const session = {
|
||||||
|
prepare: () => statement,
|
||||||
|
} as unknown as D1DatabaseSession;
|
||||||
|
return {
|
||||||
|
withSession: () => session,
|
||||||
|
} as unknown as D1Database;
|
||||||
|
}
|
||||||
|
|
||||||
|
function d1Result<T>(results: T[], changes: number): D1Result<T> {
|
||||||
|
return { success: true, results, meta: { changes } } as D1Result<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function uuid(suffix: number): string {
|
||||||
|
return `00000000-0000-4000-8000-${suffix.toString().padStart(12, "0")}`;
|
||||||
|
}
|
||||||
14
services/diagnostics-api/tsconfig.json
Normal file
14
services/diagnostics-api/tsconfig.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ES2022",
|
||||||
|
"moduleResolution": "Bundler",
|
||||||
|
"lib": ["ES2022"],
|
||||||
|
"types": ["./worker-configuration.d.ts"],
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"isolatedModules": true
|
||||||
|
},
|
||||||
|
"include": ["worker-configuration.d.ts", "src/**/*.ts"]
|
||||||
|
}
|
||||||
23
services/diagnostics-api/vitest.config.ts
Normal file
23
services/diagnostics-api/vitest.config.ts
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
import { cloudflareTest, readD1Migrations } from "@cloudflare/vitest-pool-workers";
|
||||||
|
import { dirname, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { defineConfig } from "vitest/config";
|
||||||
|
|
||||||
|
const migrationsPath = resolve(dirname(fileURLToPath(import.meta.url)), "migrations");
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
plugins: [
|
||||||
|
cloudflareTest(async () => ({
|
||||||
|
wrangler: { configPath: "./wrangler.jsonc" },
|
||||||
|
miniflare: {
|
||||||
|
bindings: {
|
||||||
|
INGEST_KEY: "test-ingest-key",
|
||||||
|
TEST_MIGRATIONS: await readD1Migrations(migrationsPath),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
],
|
||||||
|
test: {
|
||||||
|
setupFiles: ["./test/apply-migrations.ts"],
|
||||||
|
},
|
||||||
|
});
|
||||||
14719
services/diagnostics-api/worker-configuration.d.ts
vendored
Normal file
14719
services/diagnostics-api/worker-configuration.d.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
57
services/diagnostics-api/wrangler.jsonc
Normal file
57
services/diagnostics-api/wrangler.jsonc
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
{
|
||||||
|
"$schema": "./node_modules/wrangler/config-schema.json",
|
||||||
|
"name": "vnidrop-diagnostics",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"compatibility_date": "2026-07-14",
|
||||||
|
"compatibility_flags": ["nodejs_compat"],
|
||||||
|
"upload_source_maps": true,
|
||||||
|
"observability": {
|
||||||
|
"enabled": true,
|
||||||
|
"head_sampling_rate": 0.1,
|
||||||
|
},
|
||||||
|
"triggers": {
|
||||||
|
"crons": ["17 * * * *"],
|
||||||
|
},
|
||||||
|
"d1_databases": [
|
||||||
|
{
|
||||||
|
"binding": "DB",
|
||||||
|
"database_name": "vnidrop-diagnostics",
|
||||||
|
// Replace this placeholder with the ID returned by `wrangler d1 create`.
|
||||||
|
"database_id": "00000000-0000-0000-0000-000000000000",
|
||||||
|
"migrations_dir": "migrations",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"r2_buckets": [
|
||||||
|
{
|
||||||
|
"binding": "BLOBS",
|
||||||
|
"bucket_name": "vnidrop-diagnostics",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"ratelimits": [
|
||||||
|
{
|
||||||
|
"name": "INSTALL_RATE_LIMITER",
|
||||||
|
"namespace_id": "1001",
|
||||||
|
"simple": {
|
||||||
|
"limit": 30,
|
||||||
|
"period": 60,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "SOURCE_RATE_LIMITER",
|
||||||
|
"namespace_id": "1002",
|
||||||
|
"simple": {
|
||||||
|
"limit": 120,
|
||||||
|
"period": 60,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
// Optional: bind Analytics Engine as `AE` when event volume justifies it.
|
||||||
|
// "analytics_engine_datasets": [
|
||||||
|
// { "binding": "AE", "dataset": "vnidrop_events" },
|
||||||
|
// ],
|
||||||
|
"vars": {
|
||||||
|
"MAX_BODY_BYTES": "262144",
|
||||||
|
"MAX_EVENTS_PER_BATCH": "50",
|
||||||
|
"RETENTION_DAYS": "90",
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -37,32 +37,61 @@ plugins {
|
|||||||
alias(libs.plugins.kotlinAtomicfu)
|
alias(libs.plugins.kotlinAtomicfu)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compile-time switch (gradle.properties or -Pvnidrop.diagnostics.included=false).
|
// Compile-time switches (gradle.properties or -P…).
|
||||||
// false: no Share-diagnostics toggle, no telemetry/crash auto-upload stack.
|
// included=false: no Share-diagnostics toggle, no telemetry/crash auto-upload stack.
|
||||||
|
// endpoint/key both empty: transport is NoOp (safe default until Cloudflare is deployed).
|
||||||
val diagnosticsIncluded: Boolean =
|
val diagnosticsIncluded: Boolean =
|
||||||
(findProperty("vnidrop.diagnostics.included") as String?)?.toBooleanStrictOrNull() ?: true
|
(findProperty("vnidrop.diagnostics.included") as String?)?.toBooleanStrictOrNull() ?: true
|
||||||
|
val diagnosticsEndpoint: String =
|
||||||
|
(findProperty("vnidrop.diagnostics.endpoint") as String?)?.trim().orEmpty()
|
||||||
|
val diagnosticsIngestKey: String =
|
||||||
|
(findProperty("vnidrop.diagnostics.ingestKey") as String?)?.trim().orEmpty()
|
||||||
|
check(diagnosticsEndpoint.isEmpty() == diagnosticsIngestKey.isEmpty()) {
|
||||||
|
"vnidrop.diagnostics.endpoint and vnidrop.diagnostics.ingestKey must be configured together"
|
||||||
|
}
|
||||||
|
|
||||||
val diagnosticsBuildConfigDir = layout.buildDirectory.dir("generated/diagnostics/commonMain/kotlin")
|
val diagnosticsBuildConfigDir = layout.buildDirectory.dir("generated/diagnostics/commonMain/kotlin")
|
||||||
val generateDiagnosticsBuildConfig by tasks.registering {
|
val generateDiagnosticsBuildConfig by tasks.registering {
|
||||||
group = "build"
|
group = "build"
|
||||||
description = "Generates DiagnosticsBuildConfig from vnidrop.diagnostics.included"
|
description = "Generates DiagnosticsBuildConfig from vnidrop.diagnostics.* properties"
|
||||||
val outputDir = diagnosticsBuildConfigDir
|
val outputDir = diagnosticsBuildConfigDir
|
||||||
val included = diagnosticsIncluded
|
val included = diagnosticsIncluded
|
||||||
|
val endpoint = diagnosticsEndpoint
|
||||||
|
val ingestKey = diagnosticsIngestKey
|
||||||
inputs.property("vnidrop.diagnostics.included", included)
|
inputs.property("vnidrop.diagnostics.included", included)
|
||||||
|
inputs.property("vnidrop.diagnostics.endpoint", endpoint)
|
||||||
|
inputs.property("vnidrop.diagnostics.ingestKey", ingestKey)
|
||||||
outputs.dir(outputDir)
|
outputs.dir(outputDir)
|
||||||
doLast {
|
doLast {
|
||||||
val packageDir = outputDir.get().asFile.resolve("com/vnidrop/app/diagnostics")
|
val packageDir = outputDir.get().asFile.resolve("com/vnidrop/app/diagnostics")
|
||||||
packageDir.mkdirs()
|
packageDir.mkdirs()
|
||||||
|
fun esc(value: String): String = buildString {
|
||||||
|
for (ch in value) {
|
||||||
|
when (ch) {
|
||||||
|
'\\' -> append("\\\\")
|
||||||
|
'"' -> append("\\\"")
|
||||||
|
'\n' -> append("\\n")
|
||||||
|
'\r' -> append("\\r")
|
||||||
|
'\t' -> append("\\t")
|
||||||
|
'$' -> append("\\$")
|
||||||
|
else -> append(ch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
packageDir.resolve("DiagnosticsBuildConfig.kt").writeText(
|
packageDir.resolve("DiagnosticsBuildConfig.kt").writeText(
|
||||||
"""
|
"""
|
||||||
|package com.vnidrop.app.diagnostics
|
|package com.vnidrop.app.diagnostics
|
||||||
|
|
|
|
||||||
|/**
|
|/**
|
||||||
| * Generated by shared/build.gradle.kts.
|
| * Generated by shared/build.gradle.kts from vnidrop.diagnostics.* properties.
|
||||||
| * Override with `-Pvnidrop.diagnostics.included=false` or gradle.properties.
|
| *
|
||||||
|
| * - included=false → no opt-in UI / telemetry / crash hooks
|
||||||
|
| * - endpoint/key both empty → [NoOpDiagnosticsTransport] (no network)
|
||||||
| */
|
| */
|
||||||
|object DiagnosticsBuildConfig {
|
|object DiagnosticsBuildConfig {
|
||||||
| const val INCLUDED: Boolean = $included
|
| const val INCLUDED: Boolean = $included
|
||||||
|
| const val ENDPOINT: String = "${esc(endpoint)}"
|
||||||
|
| const val INGEST_KEY: String = "${esc(ingestKey)}"
|
||||||
|}
|
|}
|
||||||
|
|
|
|
||||||
""".trimMargin(),
|
""".trimMargin(),
|
||||||
|
|||||||
@@ -13,8 +13,16 @@ private class AndroidPendingCrashStore(
|
|||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
override fun write(report: CrashReport) {
|
override fun write(report: CrashReport) {
|
||||||
|
if (!isValidDiagnosticId(report.id)) return
|
||||||
directory.mkdirs()
|
directory.mkdirs()
|
||||||
File(directory, "${report.id}.crash").writeText(CrashReportCodec.encode(report), StandardCharsets.UTF_8)
|
val target = File(directory, "${report.id}.crash")
|
||||||
|
val temporary = File(directory, ".${report.id}.tmp")
|
||||||
|
val payload = CrashReportCodec.encode(report)
|
||||||
|
temporary.writeText(payload, StandardCharsets.UTF_8)
|
||||||
|
if (!temporary.renameTo(target)) {
|
||||||
|
target.writeText(payload, StandardCharsets.UTF_8)
|
||||||
|
temporary.delete()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
@@ -31,6 +39,34 @@ private class AndroidPendingCrashStore(
|
|||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
override fun delete(id: String) {
|
override fun delete(id: String) {
|
||||||
|
if (!isValidDiagnosticId(id)) return
|
||||||
File(directory, "$id.crash").delete()
|
File(directory, "$id.crash").delete()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
|
||||||
|
require(maxCount > 0) { "maxCount must be positive" }
|
||||||
|
if (!directory.isDirectory) return
|
||||||
|
directory.listFiles { file -> file.isFile && file.name.endsWith(".tmp") }
|
||||||
|
.orEmpty()
|
||||||
|
.forEach(File::delete)
|
||||||
|
val reports = directory
|
||||||
|
.listFiles { file -> file.isFile && file.name.endsWith(".crash") }
|
||||||
|
.orEmpty()
|
||||||
|
.mapNotNull { file ->
|
||||||
|
val report = runCatching {
|
||||||
|
CrashReportCodec.decode(file.readText(StandardCharsets.UTF_8))
|
||||||
|
}.getOrNull()
|
||||||
|
if (report == null) {
|
||||||
|
file.delete()
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
file to report
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.sortedByDescending { (_, report) -> report.timestampMillis }
|
||||||
|
reports.forEachIndexed { index, (file, report) ->
|
||||||
|
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) file.delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import java.io.BufferedReader
|
||||||
|
import java.io.InputStreamReader
|
||||||
|
import java.net.HttpURLConnection
|
||||||
|
import java.net.URI
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
|
||||||
|
actual suspend fun platformHttpPost(
|
||||||
|
url: String,
|
||||||
|
headers: Map<String, String>,
|
||||||
|
bodyUtf8: String,
|
||||||
|
): PlatformHttpResponse = withContext(Dispatchers.IO) {
|
||||||
|
val connection = (URI(url).toURL().openConnection() as HttpURLConnection).apply {
|
||||||
|
requestMethod = "POST"
|
||||||
|
doOutput = true
|
||||||
|
connectTimeout = 15_000
|
||||||
|
readTimeout = 30_000
|
||||||
|
setRequestProperty("Content-Type", "application/json; charset=utf-8")
|
||||||
|
headers.forEach { (key, value) -> setRequestProperty(key, value) }
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
connection.outputStream.use { output ->
|
||||||
|
output.write(bodyUtf8.toByteArray(StandardCharsets.UTF_8))
|
||||||
|
}
|
||||||
|
val code = connection.responseCode
|
||||||
|
val stream = if (code in 200..299) connection.inputStream else connection.errorStream
|
||||||
|
val body = stream?.use { input ->
|
||||||
|
BufferedReader(InputStreamReader(input, StandardCharsets.UTF_8)).readText()
|
||||||
|
}.orEmpty()
|
||||||
|
PlatformHttpResponse(code, body)
|
||||||
|
} finally {
|
||||||
|
connection.disconnect()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ package com.vnidrop.app
|
|||||||
import com.vnidrop.app.core.CoreGateway
|
import com.vnidrop.app.core.CoreGateway
|
||||||
import com.vnidrop.app.core.CoreRepository
|
import com.vnidrop.app.core.CoreRepository
|
||||||
import com.vnidrop.app.diagnostics.DiagnosticsCoordinator
|
import com.vnidrop.app.diagnostics.DiagnosticsCoordinator
|
||||||
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport
|
import com.vnidrop.app.diagnostics.createDiagnosticsTransport
|
||||||
import com.vnidrop.app.feature.approvals.ApprovalCoordinator
|
import com.vnidrop.app.feature.approvals.ApprovalCoordinator
|
||||||
import com.vnidrop.app.feature.send.AppFilePreviewRepository
|
import com.vnidrop.app.feature.send.AppFilePreviewRepository
|
||||||
import com.vnidrop.app.feature.send.createPlatformPreviewStore
|
import com.vnidrop.app.feature.send.createPlatformPreviewStore
|
||||||
@@ -18,6 +18,9 @@ import kotlinx.coroutines.CoroutineScope
|
|||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.SupervisorJob
|
import kotlinx.coroutines.SupervisorJob
|
||||||
import kotlinx.coroutines.cancel
|
import kotlinx.coroutines.cancel
|
||||||
|
import kotlinx.coroutines.flow.drop
|
||||||
|
import kotlinx.coroutines.flow.filter
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
|
|
||||||
class AppGraph(
|
class AppGraph(
|
||||||
val dependencies: AppDependencies,
|
val dependencies: AppDependencies,
|
||||||
@@ -45,7 +48,11 @@ class AppGraph(
|
|||||||
platform = dependencies.environment.name,
|
platform = dependencies.environment.name,
|
||||||
preferencesRepository = preferencesRepository,
|
preferencesRepository = preferencesRepository,
|
||||||
scope = applicationScope,
|
scope = applicationScope,
|
||||||
transport = NoOpDiagnosticsTransport(),
|
transport = createDiagnosticsTransport(
|
||||||
|
appVersion = dependencies.environment.appVersion,
|
||||||
|
platform = dependencies.environment.name,
|
||||||
|
installIdProvider = { preferencesRepository.ensureDiagnosticsInstallId() },
|
||||||
|
),
|
||||||
)
|
)
|
||||||
val approvalCoordinator = ApprovalCoordinator(
|
val approvalCoordinator = ApprovalCoordinator(
|
||||||
repository = coreRepository,
|
repository = coreRepository,
|
||||||
@@ -59,6 +66,12 @@ class AppGraph(
|
|||||||
init {
|
init {
|
||||||
AppLogger.initialize(dependencies.environment.defaultCoreDataDir)
|
AppLogger.initialize(dependencies.environment.defaultCoreDataDir)
|
||||||
diagnostics.start()
|
diagnostics.start()
|
||||||
|
applicationScope.launch {
|
||||||
|
visibility.isForeground
|
||||||
|
.drop(1)
|
||||||
|
.filter { isForeground -> !isForeground }
|
||||||
|
.collect { diagnostics.telemetry.flush() }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun close() {
|
fun close() {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package com.vnidrop.app.diagnostics
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
import com.vnidrop.app.logging.platformNowMillis
|
import com.vnidrop.app.logging.platformNowMillis
|
||||||
|
import kotlin.concurrent.atomics.AtomicReference
|
||||||
|
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fixed-size ring of high-level app breadcrumbs for crash / bug context.
|
* Fixed-size ring of high-level app breadcrumbs for crash / bug context.
|
||||||
@@ -9,6 +11,7 @@ import com.vnidrop.app.logging.platformNowMillis
|
|||||||
* Updates are best-effort under concurrency; losing a breadcrumb is preferable
|
* Updates are best-effort under concurrency; losing a breadcrumb is preferable
|
||||||
* to blocking a dying process on a lock.
|
* to blocking a dying process on a lock.
|
||||||
*/
|
*/
|
||||||
|
@OptIn(ExperimentalAtomicApi::class)
|
||||||
class BreadcrumbBuffer(
|
class BreadcrumbBuffer(
|
||||||
private val capacity: Int = DefaultCapacity,
|
private val capacity: Int = DefaultCapacity,
|
||||||
) {
|
) {
|
||||||
@@ -16,28 +19,29 @@ class BreadcrumbBuffer(
|
|||||||
require(capacity > 0) { "capacity must be positive" }
|
require(capacity > 0) { "capacity must be positive" }
|
||||||
}
|
}
|
||||||
|
|
||||||
@Volatile
|
private val items = AtomicReference<List<Breadcrumb>>(emptyList())
|
||||||
private var items: List<Breadcrumb> = emptyList()
|
|
||||||
|
|
||||||
fun add(name: String, properties: Map<String, String> = emptyMap(), timestampMillis: Long = platformNowMillis()) {
|
fun add(name: String, properties: Map<String, String> = emptyMap(), timestampMillis: Long = platformNowMillis()) {
|
||||||
|
val sanitizedName = sanitizeDiagnosticName(name)
|
||||||
|
if (sanitizedName.isBlank()) return
|
||||||
val crumb = Breadcrumb(
|
val crumb = Breadcrumb(
|
||||||
name = name.take(MaxNameLength),
|
name = sanitizedName,
|
||||||
timestampMillis = timestampMillis,
|
timestampMillis = timestampMillis,
|
||||||
properties = LogRedactor.redactMap(properties).mapValues { it.value.take(MaxPropertyValueLength) },
|
properties = sanitizeDiagnosticProperties(properties),
|
||||||
)
|
)
|
||||||
val current = items
|
while (true) {
|
||||||
items = (current + crumb).takeLast(capacity)
|
val current = items.load()
|
||||||
|
if (items.compareAndSet(current, (current + crumb).takeLast(capacity))) return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun snapshot(): List<Breadcrumb> = items
|
fun snapshot(): List<Breadcrumb> = items.load()
|
||||||
|
|
||||||
fun clear() {
|
fun clear() {
|
||||||
items = emptyList()
|
items.store(emptyList())
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
const val DefaultCapacity = 40
|
const val DefaultCapacity = 40
|
||||||
private const val MaxNameLength = 64
|
|
||||||
private const val MaxPropertyValueLength = 128
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.vnidrop.app.logging.AppLogger
|
|||||||
import com.vnidrop.app.logging.platformNowMillis
|
import com.vnidrop.app.logging.platformNowMillis
|
||||||
import com.vnidrop.app.preferences.PreferencesRepository
|
import com.vnidrop.app.preferences.PreferencesRepository
|
||||||
import com.vnidrop.app.util.randomUuidString
|
import com.vnidrop.app.util.randomUuidString
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
|
|
||||||
data class BugReportDraft(
|
data class BugReportDraft(
|
||||||
val whatHappened: String,
|
val whatHappened: String,
|
||||||
@@ -24,7 +25,7 @@ class BugReportService(
|
|||||||
private val appVersion: String,
|
private val appVersion: String,
|
||||||
private val platform: String,
|
private val platform: String,
|
||||||
private val logReader: () -> String = {
|
private val logReader: () -> String = {
|
||||||
LogRedactor.redact(AppLogger.readLatestLogs(AppLogger.DefaultBugReportLogBytes))
|
AppLogger.readLatestLogs(AppLogger.DefaultBugReportLogBytes)
|
||||||
},
|
},
|
||||||
) {
|
) {
|
||||||
fun assemble(
|
fun assemble(
|
||||||
@@ -32,25 +33,25 @@ class BugReportService(
|
|||||||
deviceInfo: DeviceInfo?,
|
deviceInfo: DeviceInfo?,
|
||||||
installId: String,
|
installId: String,
|
||||||
): BugReport {
|
): BugReport {
|
||||||
val logs = if (draft.includeLogs) logReader() else ""
|
val logs = if (draft.includeLogs) readReportLogs() else ""
|
||||||
return BugReport(
|
return BugReport(
|
||||||
id = randomUuidString(),
|
id = randomUuidString(),
|
||||||
timestampMillis = platformNowMillis(),
|
timestampMillis = platformNowMillis(),
|
||||||
installId = installId,
|
installId = sanitizeDiagnosticsInstallId(installId),
|
||||||
appVersion = appVersion,
|
appVersion = appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
|
||||||
platform = platform,
|
platform = platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
|
||||||
whatHappened = draft.whatHappened.trim().take(MaxFieldLength),
|
whatHappened = draft.whatHappened.trim().takeUtf8Bytes(MaxFieldBytes),
|
||||||
expected = draft.expected.trim().take(MaxFieldLength),
|
expected = draft.expected.trim().takeUtf8Bytes(MaxFieldBytes),
|
||||||
steps = draft.steps.trim().take(MaxFieldLength),
|
steps = draft.steps.trim().takeUtf8Bytes(MaxFieldBytes),
|
||||||
contact = draft.contact.trim().take(MaxContactLength),
|
contact = draft.contact.trim().takeUtf8Bytes(MaxContactBytes),
|
||||||
includeLogs = draft.includeLogs,
|
includeLogs = draft.includeLogs,
|
||||||
logs = logs,
|
logs = logs,
|
||||||
device = DeviceSnapshot(
|
device = DeviceSnapshot(
|
||||||
deviceName = deviceInfo?.deviceName,
|
deviceName = deviceInfo?.deviceName?.takeUtf8Bytes(128),
|
||||||
deviceModel = deviceInfo?.deviceModel,
|
deviceModel = deviceInfo?.deviceModel?.takeUtf8Bytes(128),
|
||||||
operatingSystem = deviceInfo?.operatingSystem ?: platform,
|
operatingSystem = (deviceInfo?.operatingSystem ?: platform).takeUtf8Bytes(192),
|
||||||
network = deviceInfo?.network,
|
network = deviceInfo?.network?.takeUtf8Bytes(96),
|
||||||
batteryLevel = deviceInfo?.batteryLevel,
|
batteryLevel = deviceInfo?.batteryLevel?.takeUtf8Bytes(64),
|
||||||
),
|
),
|
||||||
breadcrumbs = breadcrumbs.snapshot(),
|
breadcrumbs = breadcrumbs.snapshot(),
|
||||||
)
|
)
|
||||||
@@ -67,7 +68,13 @@ class BugReportService(
|
|||||||
}
|
}
|
||||||
val installId = preferencesRepository.ensureDiagnosticsInstallId()
|
val installId = preferencesRepository.ensureDiagnosticsInstallId()
|
||||||
val report = assemble(draft, deviceInfo, installId)
|
val report = assemble(draft, deviceInfo, installId)
|
||||||
val send = transport.sendBugReport(report)
|
val send = try {
|
||||||
|
transport.sendBugReport(report)
|
||||||
|
} catch (cancelled: CancellationException) {
|
||||||
|
throw cancelled
|
||||||
|
} catch (error: Throwable) {
|
||||||
|
Result.failure(error)
|
||||||
|
}
|
||||||
return send.fold(
|
return send.fold(
|
||||||
onSuccess = {
|
onSuccess = {
|
||||||
AppLogger.info("bug_report", "submitted", mapOf("id" to report.id))
|
AppLogger.info("bug_report", "submitted", mapOf("id" to report.id))
|
||||||
@@ -80,10 +87,14 @@ class BugReportService(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun previewLogBytes(): Int = logReader().encodeToByteArray().size
|
fun previewLogBytes(): Int = readReportLogs().encodeToByteArray().size
|
||||||
|
|
||||||
|
private fun readReportLogs(): String =
|
||||||
|
LogRedactor.redact(logReader()).takeUtf8Bytes(MaxLogBytes)
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private const val MaxFieldLength = 4_000
|
internal const val MaxLogBytes = 192 * 1024
|
||||||
private const val MaxContactLength = 320
|
private const val MaxFieldBytes = 4_000
|
||||||
|
private const val MaxContactBytes = 320
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,13 +5,18 @@ import com.vnidrop.app.logging.platformNowMillis
|
|||||||
import com.vnidrop.app.preferences.PreferencesRepository
|
import com.vnidrop.app.preferences.PreferencesRepository
|
||||||
import com.vnidrop.app.util.randomUuidString
|
import com.vnidrop.app.util.randomUuidString
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
import kotlinx.coroutines.flow.first
|
import kotlinx.coroutines.flow.first
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
import kotlin.concurrent.atomics.AtomicBoolean
|
||||||
|
import kotlin.concurrent.atomics.AtomicReference
|
||||||
|
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Captures uncaught exceptions to disk, then uploads on a later launch when
|
* Captures uncaught exceptions to disk, then uploads on a later launch when
|
||||||
* diagnostics is enabled (and when a real [DiagnosticsTransport] is wired).
|
* diagnostics is enabled (and when a real [DiagnosticsTransport] is wired).
|
||||||
*/
|
*/
|
||||||
|
@OptIn(ExperimentalAtomicApi::class)
|
||||||
class CrashReporter(
|
class CrashReporter(
|
||||||
private val store: PendingCrashStore,
|
private val store: PendingCrashStore,
|
||||||
private val preferencesRepository: PreferencesRepository,
|
private val preferencesRepository: PreferencesRepository,
|
||||||
@@ -21,69 +26,133 @@ class CrashReporter(
|
|||||||
private val platform: String,
|
private val platform: String,
|
||||||
private val scope: CoroutineScope,
|
private val scope: CoroutineScope,
|
||||||
) {
|
) {
|
||||||
@Volatile private var installed = false
|
private val installed = AtomicBoolean(false)
|
||||||
@Volatile private var lastInstallId: String = ""
|
private val observingPreferences = AtomicBoolean(false)
|
||||||
@Volatile private var lastDiagnosticsEnabled: Boolean = false
|
private val capturePolicy = AtomicReference(CrashCapturePolicy())
|
||||||
|
|
||||||
fun startObservingPreferences() {
|
fun startObservingPreferences() {
|
||||||
|
if (!observingPreferences.compareAndSet(false, true)) return
|
||||||
scope.launch {
|
scope.launch {
|
||||||
preferencesRepository.preferences.collect { prefs ->
|
preferencesRepository.preferences.collect { prefs ->
|
||||||
lastInstallId = prefs.diagnosticsInstallId
|
capturePolicy.store(
|
||||||
lastDiagnosticsEnabled = prefs.diagnosticsEnabled
|
CrashCapturePolicy(
|
||||||
|
installId = prefs.diagnosticsInstallId,
|
||||||
|
diagnosticsEnabled = prefs.diagnosticsEnabled,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if (!prefs.diagnosticsEnabled) {
|
||||||
|
runCatching(::deleteAllPending)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun installUnhandledExceptionHandler() {
|
fun installUnhandledExceptionHandler() {
|
||||||
if (!DiagnosticsBuildConfig.INCLUDED) return
|
if (!DiagnosticsBuildConfig.INCLUDED) return
|
||||||
if (installed) return
|
if (!installed.compareAndSet(false, true)) return
|
||||||
installed = true
|
|
||||||
installPlatformCrashHook { throwable ->
|
installPlatformCrashHook { throwable ->
|
||||||
capture(throwable)
|
capture(throwable)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun capture(throwable: Throwable, diagnosticsEnabledOverride: Boolean? = null): CrashReport {
|
fun capture(throwable: Throwable, diagnosticsEnabledOverride: Boolean? = null): CrashReport {
|
||||||
|
val policy = capturePolicy.load()
|
||||||
val report = CrashReport(
|
val report = CrashReport(
|
||||||
id = randomUuidString(),
|
id = randomUuidString(),
|
||||||
timestampMillis = platformNowMillis(),
|
timestampMillis = platformNowMillis(),
|
||||||
installId = lastInstallId,
|
installId = sanitizeDiagnosticsInstallId(policy.installId),
|
||||||
appVersion = appVersion,
|
appVersion = appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
|
||||||
platform = platform,
|
platform = platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
|
||||||
exceptionType = throwable::class.simpleName ?: "Throwable",
|
exceptionType = throwable::class.simpleName ?: "Throwable",
|
||||||
exceptionMessage = LogRedactor.redact(throwable.message.orEmpty()).take(MaxMessageLength),
|
exceptionMessage = LogRedactor.redact(throwable.message.orEmpty()).takeUtf8Bytes(MaxMessageBytes),
|
||||||
stackTrace = LogRedactor.redact(throwable.stackTraceToString()).take(MaxStackLength),
|
stackTrace = LogRedactor.redact(throwable.stackTraceToString()).takeUtf8Bytes(MaxStackBytes),
|
||||||
breadcrumbs = breadcrumbs.snapshot(),
|
breadcrumbs = breadcrumbs.snapshot(),
|
||||||
diagnosticsEnabledAtCapture = diagnosticsEnabledOverride ?: lastDiagnosticsEnabled,
|
diagnosticsEnabledAtCapture = diagnosticsEnabledOverride ?: policy.diagnosticsEnabled,
|
||||||
)
|
)
|
||||||
|
if (report.diagnosticsEnabledAtCapture != false) {
|
||||||
runCatching { store.write(report) }
|
runCatching { store.write(report) }
|
||||||
|
runCatching {
|
||||||
|
store.prune(
|
||||||
|
olderThanTimestampMillis = platformNowMillis() - LocalRetentionMillis,
|
||||||
|
maxCount = MaxLocalCrashCount,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
AppLogger.error("crash", "captured crash ${report.id}", throwable)
|
AppLogger.error("crash", "captured crash ${report.id}", throwable)
|
||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Uploads pending crashes that were captured with diagnostics enabled.
|
* Uploads pending crashes that were captured with diagnostics enabled.
|
||||||
* Local files are always kept for bug-report attachment until deleted after successful send.
|
* Local files are deleted after successful delivery or bounded by local retention.
|
||||||
*/
|
*/
|
||||||
suspend fun flushPending() {
|
suspend fun flushPending() {
|
||||||
val diagnosticsEnabled = preferencesRepository.preferences.first().diagnosticsEnabled
|
store.prune(
|
||||||
if (!diagnosticsEnabled) return
|
olderThanTimestampMillis = platformNowMillis() - LocalRetentionMillis,
|
||||||
|
maxCount = MaxLocalCrashCount,
|
||||||
|
)
|
||||||
for (report in store.list()) {
|
for (report in store.list()) {
|
||||||
// Only auto-upload crashes captured while diagnostics was on.
|
val preferences = preferencesRepository.preferences.first()
|
||||||
if (!report.diagnosticsEnabledAtCapture) continue
|
if (!preferences.diagnosticsEnabled || capturePolicy.load().diagnosticsEnabled == false) {
|
||||||
val result = transport.sendCrash(report)
|
deleteAllPending()
|
||||||
if (result.isSuccess) {
|
return
|
||||||
store.delete(report.id)
|
|
||||||
}
|
}
|
||||||
|
if (report.diagnosticsEnabledAtCapture == false) {
|
||||||
|
store.delete(report.id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val installId = sanitizeDiagnosticsInstallId(
|
||||||
|
preferences.diagnosticsInstallId.ifBlank {
|
||||||
|
preferencesRepository.ensureDiagnosticsInstallId()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
val resolved = report.copy(
|
||||||
|
installId = report.installId.ifBlank { installId },
|
||||||
|
diagnosticsEnabledAtCapture = true,
|
||||||
|
)
|
||||||
|
if (resolved != report) store.write(resolved)
|
||||||
|
if (
|
||||||
|
!preferencesRepository.preferences.first().diagnosticsEnabled ||
|
||||||
|
capturePolicy.load().diagnosticsEnabled == false
|
||||||
|
) {
|
||||||
|
deleteAllPending()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
val result = try {
|
||||||
|
transport.sendCrash(resolved)
|
||||||
|
} catch (cancelled: CancellationException) {
|
||||||
|
throw cancelled
|
||||||
|
} catch (error: Throwable) {
|
||||||
|
Result.failure(error)
|
||||||
|
}
|
||||||
|
if (result.isSuccess) {
|
||||||
|
store.delete(resolved.id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val error = result.exceptionOrNull()
|
||||||
|
if (error?.isPermanentDiagnosticsPayloadRejection() == true) {
|
||||||
|
store.delete(resolved.id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun latestLocalCrash(): CrashReport? = store.list().maxByOrNull { it.timestampMillis }
|
private fun deleteAllPending() {
|
||||||
|
store.list().forEach { report -> store.delete(report.id) }
|
||||||
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private const val MaxMessageLength = 2_000
|
private const val MaxMessageBytes = 2_000
|
||||||
private const val MaxStackLength = 32_000
|
private const val MaxStackBytes = 32_000
|
||||||
|
private const val MaxLocalCrashCount = 20
|
||||||
|
private const val LocalRetentionMillis = 30L * 86_400_000L
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private data class CrashCapturePolicy(
|
||||||
|
val installId: String = "",
|
||||||
|
val diagnosticsEnabled: Boolean? = null,
|
||||||
|
)
|
||||||
|
|
||||||
expect fun installPlatformCrashHook(onCrash: (Throwable) -> Unit)
|
expect fun installPlatformCrashHook(onCrash: (Throwable) -> Unit)
|
||||||
|
|||||||
@@ -0,0 +1,221 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimal JSON encoding for diagnostics payloads (no kotlinx.serialization dependency).
|
||||||
|
*/
|
||||||
|
internal object DiagnosticsJson {
|
||||||
|
internal const val MaxRequestBytes = 256 * 1024
|
||||||
|
internal const val MaxInstallIdBytes = 80
|
||||||
|
internal const val MaxAppVersionBytes = 40
|
||||||
|
internal const val MaxPlatformBytes = 40
|
||||||
|
private const val MaxBreadcrumbsJsonBytes = 16_000
|
||||||
|
private const val MaxBreadcrumbs = 40
|
||||||
|
private const val SizedBatchId = "00000000-0000-4000-8000-000000000000"
|
||||||
|
|
||||||
|
fun eventsBody(
|
||||||
|
batchId: String,
|
||||||
|
installId: String,
|
||||||
|
appVersion: String,
|
||||||
|
platform: String,
|
||||||
|
events: List<TelemetryEvent>,
|
||||||
|
): String = buildString {
|
||||||
|
append('{')
|
||||||
|
appendJsonField("batchId", batchId)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("installId", installId)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("appVersion", appVersion)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("platform", platform)
|
||||||
|
append(',')
|
||||||
|
append("\"events\":[")
|
||||||
|
events.forEachIndexed { index, event ->
|
||||||
|
if (index > 0) append(',')
|
||||||
|
append('{')
|
||||||
|
appendJsonField("name", event.name)
|
||||||
|
append(',')
|
||||||
|
append("\"timestampMillis\":")
|
||||||
|
append(event.timestampMillis)
|
||||||
|
append(',')
|
||||||
|
append("\"schemaVersion\":")
|
||||||
|
append(event.schemaVersion)
|
||||||
|
append(',')
|
||||||
|
append("\"properties\":")
|
||||||
|
appendStringMap(event.properties)
|
||||||
|
append('}')
|
||||||
|
}
|
||||||
|
append("]}")
|
||||||
|
}
|
||||||
|
|
||||||
|
fun eventBatchFitsRequest(events: List<TelemetryEvent>): Boolean =
|
||||||
|
eventsBody(
|
||||||
|
batchId = SizedBatchId,
|
||||||
|
installId = "\u0000".repeat(MaxInstallIdBytes),
|
||||||
|
appVersion = "\u0000".repeat(MaxAppVersionBytes),
|
||||||
|
platform = "\u0000".repeat(MaxPlatformBytes),
|
||||||
|
events = events,
|
||||||
|
).encodeToByteArray().size <= MaxRequestBytes
|
||||||
|
|
||||||
|
fun crashBody(report: CrashReport): String = buildString {
|
||||||
|
append('{')
|
||||||
|
appendJsonField("id", report.id)
|
||||||
|
append(',')
|
||||||
|
append("\"timestampMillis\":")
|
||||||
|
append(report.timestampMillis)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("installId", report.installId)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("appVersion", report.appVersion)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("platform", report.platform)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("exceptionType", report.exceptionType)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("exceptionMessage", report.exceptionMessage)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("stackTrace", report.stackTrace)
|
||||||
|
append(',')
|
||||||
|
append("\"diagnosticsEnabledAtCapture\":")
|
||||||
|
append(requireNotNull(report.diagnosticsEnabledAtCapture) {
|
||||||
|
"crash consent must be resolved before delivery"
|
||||||
|
})
|
||||||
|
append(',')
|
||||||
|
append("\"schemaVersion\":")
|
||||||
|
append(report.schemaVersion)
|
||||||
|
append(',')
|
||||||
|
append("\"breadcrumbs\":")
|
||||||
|
appendBreadcrumbs(report.breadcrumbs)
|
||||||
|
append('}')
|
||||||
|
}
|
||||||
|
|
||||||
|
fun bugBody(report: BugReport): String {
|
||||||
|
val logs = if (report.includeLogs) report.logs else ""
|
||||||
|
val complete = buildBugBody(report, logs)
|
||||||
|
if (complete.encodeToByteArray().size <= MaxRequestBytes || logs.isEmpty()) return complete
|
||||||
|
|
||||||
|
var best = buildBugBody(report, "")
|
||||||
|
if (best.encodeToByteArray().size > MaxRequestBytes) return best
|
||||||
|
var minimumBytes = 0
|
||||||
|
var maximumBytes = logs.encodeToByteArray().size
|
||||||
|
while (minimumBytes <= maximumBytes) {
|
||||||
|
val candidateBytes = minimumBytes + (maximumBytes - minimumBytes) / 2
|
||||||
|
val candidate = buildBugBody(report, logs.takeUtf8Bytes(candidateBytes))
|
||||||
|
if (candidate.encodeToByteArray().size <= MaxRequestBytes) {
|
||||||
|
best = candidate
|
||||||
|
minimumBytes = candidateBytes + 1
|
||||||
|
} else {
|
||||||
|
maximumBytes = candidateBytes - 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun buildBugBody(report: BugReport, logs: String): String = buildString {
|
||||||
|
append('{')
|
||||||
|
appendJsonField("id", report.id)
|
||||||
|
append(',')
|
||||||
|
append("\"timestampMillis\":")
|
||||||
|
append(report.timestampMillis)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("installId", report.installId)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("appVersion", report.appVersion)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("platform", report.platform)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("whatHappened", report.whatHappened)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("expected", report.expected)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("steps", report.steps)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("contact", report.contact)
|
||||||
|
append(',')
|
||||||
|
append("\"includeLogs\":")
|
||||||
|
append(report.includeLogs)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("logs", logs)
|
||||||
|
append(',')
|
||||||
|
append("\"schemaVersion\":")
|
||||||
|
append(report.schemaVersion)
|
||||||
|
append(',')
|
||||||
|
append("\"device\":{")
|
||||||
|
appendJsonField("deviceName", report.device.deviceName.orEmpty())
|
||||||
|
append(',')
|
||||||
|
appendJsonField("deviceModel", report.device.deviceModel.orEmpty())
|
||||||
|
append(',')
|
||||||
|
appendJsonField("operatingSystem", report.device.operatingSystem)
|
||||||
|
append(',')
|
||||||
|
appendJsonField("network", report.device.network.orEmpty())
|
||||||
|
append(',')
|
||||||
|
appendJsonField("batteryLevel", report.device.batteryLevel.orEmpty())
|
||||||
|
append("},")
|
||||||
|
append("\"breadcrumbs\":")
|
||||||
|
appendBreadcrumbs(report.breadcrumbs)
|
||||||
|
append('}')
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun StringBuilder.appendBreadcrumbs(crumbs: List<Breadcrumb>) {
|
||||||
|
append('[')
|
||||||
|
var encodedBytes = 2
|
||||||
|
var appended = 0
|
||||||
|
for (crumb in crumbs) {
|
||||||
|
if (appended == MaxBreadcrumbs) break
|
||||||
|
val name = sanitizeDiagnosticName(crumb.name)
|
||||||
|
if (name.isBlank() || crumb.timestampMillis < 0) continue
|
||||||
|
val encoded = buildString {
|
||||||
|
append('{')
|
||||||
|
appendJsonField("name", name)
|
||||||
|
append(',')
|
||||||
|
append("\"timestampMillis\":")
|
||||||
|
append(crumb.timestampMillis)
|
||||||
|
append(',')
|
||||||
|
append("\"properties\":")
|
||||||
|
appendStringMap(crumb.properties)
|
||||||
|
append('}')
|
||||||
|
}
|
||||||
|
val additionBytes = encoded.encodeToByteArray().size + if (appended == 0) 0 else 1
|
||||||
|
if (encodedBytes + additionBytes > MaxBreadcrumbsJsonBytes) break
|
||||||
|
if (appended > 0) append(',')
|
||||||
|
append(encoded)
|
||||||
|
encodedBytes += additionBytes
|
||||||
|
appended += 1
|
||||||
|
}
|
||||||
|
append(']')
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun StringBuilder.appendStringMap(map: Map<String, String>) {
|
||||||
|
append('{')
|
||||||
|
sanitizeDiagnosticProperties(map).entries.forEachIndexed { index, (key, value) ->
|
||||||
|
if (index > 0) append(',')
|
||||||
|
appendJsonField(key, value)
|
||||||
|
}
|
||||||
|
append('}')
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun StringBuilder.appendJsonField(key: String, value: String) {
|
||||||
|
append('"')
|
||||||
|
append(escape(key))
|
||||||
|
append("\":\"")
|
||||||
|
append(escape(value))
|
||||||
|
append('"')
|
||||||
|
}
|
||||||
|
|
||||||
|
internal fun escape(raw: String): String = buildString(raw.length + 8) {
|
||||||
|
for (ch in raw) {
|
||||||
|
when (ch) {
|
||||||
|
'\\' -> append("\\\\")
|
||||||
|
'"' -> append("\\\"")
|
||||||
|
'\n' -> append("\\n")
|
||||||
|
'\r' -> append("\\r")
|
||||||
|
'\t' -> append("\\t")
|
||||||
|
else -> if (ch.code < 0x20) {
|
||||||
|
append("\\u")
|
||||||
|
append(ch.code.toString(16).padStart(4, '0'))
|
||||||
|
} else {
|
||||||
|
append(ch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,12 @@ data class TelemetryEvent(
|
|||||||
val schemaVersion: Int = DiagnosticsSchemaVersion,
|
val schemaVersion: Int = DiagnosticsSchemaVersion,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/** One idempotent upload unit; [id] remains stable when delivery is retried. */
|
||||||
|
data class TelemetryBatch(
|
||||||
|
val id: String,
|
||||||
|
val events: List<TelemetryEvent>,
|
||||||
|
)
|
||||||
|
|
||||||
data class Breadcrumb(
|
data class Breadcrumb(
|
||||||
val name: String,
|
val name: String,
|
||||||
val timestampMillis: Long,
|
val timestampMillis: Long,
|
||||||
@@ -28,8 +34,8 @@ data class CrashReport(
|
|||||||
val exceptionMessage: String,
|
val exceptionMessage: String,
|
||||||
val stackTrace: String,
|
val stackTrace: String,
|
||||||
val breadcrumbs: List<Breadcrumb>,
|
val breadcrumbs: List<Breadcrumb>,
|
||||||
/** Whether diagnostics was opted in when the crash was captured. */
|
/** `null` only while a startup crash is waiting for the persisted preference to load. */
|
||||||
val diagnosticsEnabledAtCapture: Boolean,
|
val diagnosticsEnabledAtCapture: Boolean?,
|
||||||
val schemaVersion: Int = DiagnosticsSchemaVersion,
|
val schemaVersion: Int = DiagnosticsSchemaVersion,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
internal const val MaxDiagnosticProperties = 12
|
||||||
|
internal const val MaxDiagnosticPropertyKeyBytes = 40
|
||||||
|
internal const val MaxDiagnosticPropertyValueBytes = 128
|
||||||
|
internal const val MaxDiagnosticNameBytes = 64
|
||||||
|
|
||||||
|
internal fun sanitizeDiagnosticsInstallId(value: String): String {
|
||||||
|
val trimmed = value.trim()
|
||||||
|
if (trimmed.any { it.code < 0x20 || it.code == 0x7f }) return ""
|
||||||
|
return trimmed.takeUtf8Bytes(DiagnosticsJson.MaxInstallIdBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
internal fun sanitizeDiagnosticName(name: String): String =
|
||||||
|
name.takeUtf8Bytes(MaxDiagnosticNameBytes)
|
||||||
|
|
||||||
|
internal fun sanitizeDiagnosticProperties(properties: Map<String, String>): Map<String, String> {
|
||||||
|
val sanitized = LinkedHashMap<String, String>(minOf(properties.size, MaxDiagnosticProperties))
|
||||||
|
for ((rawKey, rawValue) in properties) {
|
||||||
|
val key = rawKey.takeUtf8Bytes(MaxDiagnosticPropertyKeyBytes)
|
||||||
|
if (key.isEmpty() || key in sanitized) continue
|
||||||
|
sanitized[key] = LogRedactor.redact(rawValue).takeUtf8Bytes(MaxDiagnosticPropertyValueBytes)
|
||||||
|
if (sanitized.size == MaxDiagnosticProperties) break
|
||||||
|
}
|
||||||
|
return sanitized
|
||||||
|
}
|
||||||
|
|
||||||
|
internal fun String.takeUtf8Bytes(maxBytes: Int): String {
|
||||||
|
require(maxBytes >= 0) { "maxBytes must not be negative" }
|
||||||
|
val encoded = encodeToByteArray()
|
||||||
|
if (encoded.size <= maxBytes) return this
|
||||||
|
for (endIndex in maxBytes downTo (maxBytes - 3).coerceAtLeast(0)) {
|
||||||
|
val decoded = runCatching {
|
||||||
|
encoded.decodeToString(0, endIndex, throwOnInvalidSequence = true)
|
||||||
|
}.getOrNull()
|
||||||
|
if (decoded != null) return decoded
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -1,35 +1,42 @@
|
|||||||
package com.vnidrop.app.diagnostics
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
/**
|
/** Network boundary for diagnostics; keep batching and validation client-side. */
|
||||||
* Network boundary for diagnostics. Production will swap [NoOpDiagnosticsTransport]
|
|
||||||
* for a Cloudflare Worker client; keep batching/validation client-side.
|
|
||||||
*/
|
|
||||||
interface DiagnosticsTransport {
|
interface DiagnosticsTransport {
|
||||||
suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit>
|
suspend fun sendEvents(batch: TelemetryBatch): Result<Unit>
|
||||||
suspend fun sendCrash(report: CrashReport): Result<Unit>
|
suspend fun sendCrash(report: CrashReport): Result<Unit>
|
||||||
suspend fun sendBugReport(report: BugReport): Result<Unit>
|
suspend fun sendBugReport(report: BugReport): Result<Unit>
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Accepts payloads without leaving the device. Used until Cloudflare is wired. */
|
internal class DiagnosticsUnavailableException : IllegalStateException("diagnostics delivery is not configured")
|
||||||
|
|
||||||
|
internal fun Throwable.isPermanentDiagnosticsPayloadRejection(): Boolean =
|
||||||
|
this is DiagnosticsPayloadException ||
|
||||||
|
(this is DiagnosticsHttpException && statusCode in setOf(400, 413, 415, 422))
|
||||||
|
|
||||||
|
/** Fails delivery without leaving the device. Used until a remote endpoint is configured. */
|
||||||
class NoOpDiagnosticsTransport : DiagnosticsTransport {
|
class NoOpDiagnosticsTransport : DiagnosticsTransport {
|
||||||
override suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit> = Result.success(Unit)
|
override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> = unavailable()
|
||||||
override suspend fun sendCrash(report: CrashReport): Result<Unit> = Result.success(Unit)
|
override suspend fun sendCrash(report: CrashReport): Result<Unit> = unavailable()
|
||||||
override suspend fun sendBugReport(report: BugReport): Result<Unit> = Result.success(Unit)
|
override suspend fun sendBugReport(report: BugReport): Result<Unit> = unavailable()
|
||||||
|
|
||||||
|
private fun unavailable(): Result<Unit> = Result.failure(DiagnosticsUnavailableException())
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test double that records calls and can fail on demand.
|
* Test double that records calls and can fail on demand.
|
||||||
*/
|
*/
|
||||||
class RecordingDiagnosticsTransport : DiagnosticsTransport {
|
class RecordingDiagnosticsTransport : DiagnosticsTransport {
|
||||||
val events = mutableListOf<List<TelemetryEvent>>()
|
val eventBatches = mutableListOf<TelemetryBatch>()
|
||||||
|
val events: List<List<TelemetryEvent>>
|
||||||
|
get() = eventBatches.map(TelemetryBatch::events)
|
||||||
val crashes = mutableListOf<CrashReport>()
|
val crashes = mutableListOf<CrashReport>()
|
||||||
val bugReports = mutableListOf<BugReport>()
|
val bugReports = mutableListOf<BugReport>()
|
||||||
var eventsResult: Result<Unit> = Result.success(Unit)
|
var eventsResult: Result<Unit> = Result.success(Unit)
|
||||||
var crashResult: Result<Unit> = Result.success(Unit)
|
var crashResult: Result<Unit> = Result.success(Unit)
|
||||||
var bugResult: Result<Unit> = Result.success(Unit)
|
var bugResult: Result<Unit> = Result.success(Unit)
|
||||||
|
|
||||||
override suspend fun sendEvents(events: List<TelemetryEvent>): Result<Unit> {
|
override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> {
|
||||||
this.events += events
|
eventBatches += batch
|
||||||
return eventsResult
|
return eventsResult
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import com.vnidrop.app.logging.AppLogger
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
|
|
||||||
|
/**
|
||||||
|
* HTTPS client for the Cloudflare diagnostics Worker.
|
||||||
|
* No-ops are preferred when [baseUrl] is blank — see [createDiagnosticsTransport].
|
||||||
|
*/
|
||||||
|
class HttpDiagnosticsTransport(
|
||||||
|
baseUrl: String,
|
||||||
|
private val ingestKey: String,
|
||||||
|
private val appVersion: String = "",
|
||||||
|
private val platform: String = "",
|
||||||
|
private val installIdProvider: suspend () -> String = { "" },
|
||||||
|
private val post: suspend (url: String, headers: Map<String, String>, body: String) -> PlatformHttpResponse =
|
||||||
|
{ url, headers, body -> platformHttpPost(url, headers, body) },
|
||||||
|
) : DiagnosticsTransport {
|
||||||
|
private val root = baseUrl.trim().trimEnd('/')
|
||||||
|
|
||||||
|
init {
|
||||||
|
require(root.isEmpty() || root.isAllowedDiagnosticsEndpoint()) {
|
||||||
|
"diagnostics endpoint must use HTTPS unless it targets a loopback host"
|
||||||
|
}
|
||||||
|
require(root.isEmpty() || ingestKey.isNotBlank()) {
|
||||||
|
"diagnostics ingest key must be configured when the endpoint is set"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun sendEvents(batch: TelemetryBatch): Result<Unit> {
|
||||||
|
if (batch.events.isEmpty()) return Result.success(Unit)
|
||||||
|
if (batch.events.size > TelemetryRecorder.MaxEventsPerBatch) {
|
||||||
|
return Result.failure(DiagnosticsPayloadException("diagnostics event batch is too large"))
|
||||||
|
}
|
||||||
|
if (batch.events.any { it.name.isBlank() || it.timestampMillis < 0 }) {
|
||||||
|
return Result.failure(DiagnosticsPayloadException("diagnostics event batch is invalid"))
|
||||||
|
}
|
||||||
|
val installId = sanitizeDiagnosticsInstallId(installIdProvider())
|
||||||
|
val body = DiagnosticsJson.eventsBody(
|
||||||
|
batch.id,
|
||||||
|
installId,
|
||||||
|
appVersion.takeUtf8Bytes(DiagnosticsJson.MaxAppVersionBytes),
|
||||||
|
platform.takeUtf8Bytes(DiagnosticsJson.MaxPlatformBytes),
|
||||||
|
batch.events,
|
||||||
|
)
|
||||||
|
return postJson("/v1/events", body, installId, batch.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun sendCrash(report: CrashReport): Result<Unit> {
|
||||||
|
if (report.diagnosticsEnabledAtCapture == null) {
|
||||||
|
return Result.failure(DiagnosticsPayloadException("crash consent is unresolved"))
|
||||||
|
}
|
||||||
|
val body = DiagnosticsJson.crashBody(report)
|
||||||
|
return postJson("/v1/crashes", body, report.installId, report.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun sendBugReport(report: BugReport): Result<Unit> {
|
||||||
|
val body = DiagnosticsJson.bugBody(report)
|
||||||
|
return postJson("/v1/bugs", body, report.installId, report.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun postJson(
|
||||||
|
path: String,
|
||||||
|
body: String,
|
||||||
|
installId: String,
|
||||||
|
expectedId: String,
|
||||||
|
): Result<Unit> {
|
||||||
|
if (root.isEmpty()) {
|
||||||
|
return Result.failure(IllegalStateException("diagnostics endpoint is not configured"))
|
||||||
|
}
|
||||||
|
if (body.encodeToByteArray().size > DiagnosticsJson.MaxRequestBytes) {
|
||||||
|
return Result.failure(DiagnosticsPayloadException("diagnostics $path payload is too large"))
|
||||||
|
}
|
||||||
|
return try {
|
||||||
|
val response = post(
|
||||||
|
"$root$path",
|
||||||
|
mapOf(
|
||||||
|
"X-VniDrop-Key" to ingestKey,
|
||||||
|
"X-VniDrop-Install-Id" to installId,
|
||||||
|
"Accept" to "application/json",
|
||||||
|
),
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
if (response.statusCode !in 200..299) {
|
||||||
|
AppLogger.warn(
|
||||||
|
"diagnostics",
|
||||||
|
"transport rejected $path",
|
||||||
|
mapOf("status" to response.statusCode.toString()),
|
||||||
|
)
|
||||||
|
throw DiagnosticsHttpException(response.statusCode, path)
|
||||||
|
}
|
||||||
|
if (!response.body.isSuccessfulDiagnosticsAcknowledgement(expectedId)) {
|
||||||
|
throw DiagnosticsProtocolException(path)
|
||||||
|
}
|
||||||
|
Result.success(Unit)
|
||||||
|
} catch (cancelled: CancellationException) {
|
||||||
|
throw cancelled
|
||||||
|
} catch (error: Throwable) {
|
||||||
|
Result.failure(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal class DiagnosticsHttpException(
|
||||||
|
val statusCode: Int,
|
||||||
|
path: String,
|
||||||
|
) : IllegalStateException("diagnostics $path failed: HTTP $statusCode")
|
||||||
|
|
||||||
|
internal class DiagnosticsProtocolException(path: String) :
|
||||||
|
IllegalStateException("diagnostics $path returned an invalid acknowledgement")
|
||||||
|
|
||||||
|
internal class DiagnosticsPayloadException(message: String) : IllegalArgumentException(message)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds transport from compile-time config. Empty endpoint and key → [NoOpDiagnosticsTransport].
|
||||||
|
*/
|
||||||
|
fun createDiagnosticsTransport(
|
||||||
|
appVersion: String,
|
||||||
|
platform: String,
|
||||||
|
installIdProvider: suspend () -> String,
|
||||||
|
): DiagnosticsTransport {
|
||||||
|
val endpoint = DiagnosticsBuildConfig.ENDPOINT.trim()
|
||||||
|
val ingestKey = DiagnosticsBuildConfig.INGEST_KEY.trim()
|
||||||
|
if (endpoint.isEmpty() && ingestKey.isEmpty()) return NoOpDiagnosticsTransport()
|
||||||
|
check(endpoint.isNotEmpty() && ingestKey.isNotEmpty()) {
|
||||||
|
"diagnostics endpoint and ingest key must be configured together"
|
||||||
|
}
|
||||||
|
return HttpDiagnosticsTransport(
|
||||||
|
baseUrl = endpoint,
|
||||||
|
ingestKey = ingestKey,
|
||||||
|
appVersion = appVersion,
|
||||||
|
platform = platform,
|
||||||
|
installIdProvider = installIdProvider,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.isSuccessfulDiagnosticsAcknowledgement(expectedId: String): Boolean {
|
||||||
|
val json = trim()
|
||||||
|
if (!SuccessfulAcknowledgement.matches(json)) return false
|
||||||
|
if (AcknowledgementOk.findAll(json).count() != 1) return false
|
||||||
|
val ids = AcknowledgementId.findAll(json).toList()
|
||||||
|
return ids.size == 1 &&
|
||||||
|
ids.single().groupValues[1] == "\"${DiagnosticsJson.escape(expectedId.lowercase())}\""
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.isAllowedDiagnosticsEndpoint(): Boolean {
|
||||||
|
if (any(Char::isWhitespace) || '?' in this || '#' in this) return false
|
||||||
|
val schemeSeparator = indexOf("://")
|
||||||
|
if (schemeSeparator <= 0) return false
|
||||||
|
val scheme = substring(0, schemeSeparator).lowercase()
|
||||||
|
val authority = substring(schemeSeparator + 3).substringBefore('/')
|
||||||
|
if (authority.isEmpty() || '@' in authority) return false
|
||||||
|
val host = when {
|
||||||
|
authority.startsWith('[') -> {
|
||||||
|
val end = authority.indexOf(']')
|
||||||
|
if (end <= 1) return false
|
||||||
|
val suffix = authority.substring(end + 1)
|
||||||
|
if (suffix.isNotEmpty() && !suffix.isValidPortSuffix()) return false
|
||||||
|
authority.substring(1, end)
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
if (authority.count { it == ':' } > 1) return false
|
||||||
|
val portSeparator = authority.indexOf(':')
|
||||||
|
if (portSeparator >= 0 && !authority.substring(portSeparator).isValidPortSuffix()) return false
|
||||||
|
authority.substringBefore(':')
|
||||||
|
}
|
||||||
|
}.lowercase()
|
||||||
|
if (host.isEmpty()) return false
|
||||||
|
if (scheme == "https") return true
|
||||||
|
return scheme == "http" && host.isLoopbackHost()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.isValidPortSuffix(): Boolean =
|
||||||
|
startsWith(':') && drop(1).toIntOrNull() in 1..65_535
|
||||||
|
|
||||||
|
private fun String.isLoopbackHost(): Boolean {
|
||||||
|
if (this == "localhost" || this == "::1") return true
|
||||||
|
val octets = split('.')
|
||||||
|
return octets.size == 4 &&
|
||||||
|
octets.first() == "127" &&
|
||||||
|
octets.all { it.toIntOrNull() in 0..255 }
|
||||||
|
}
|
||||||
|
|
||||||
|
private const val JsonStringPattern =
|
||||||
|
""""(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*""""
|
||||||
|
private const val JsonNumberPattern =
|
||||||
|
"""-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?"""
|
||||||
|
private val SuccessfulAcknowledgement = Regex(
|
||||||
|
"""^\s*\{\s*"ok"\s*:\s*true(?:\s*,\s*$JsonStringPattern\s*:\s*(?:$JsonStringPattern|$JsonNumberPattern|true|false|null))*\s*}\s*$""",
|
||||||
|
)
|
||||||
|
private val AcknowledgementOk = Regex(""""ok"\s*:""")
|
||||||
|
private val AcknowledgementId = Regex(""""id"\s*:\s*($JsonStringPattern)""")
|
||||||
@@ -8,6 +8,7 @@ interface PendingCrashStore {
|
|||||||
fun write(report: CrashReport)
|
fun write(report: CrashReport)
|
||||||
fun list(): List<CrashReport>
|
fun list(): List<CrashReport>
|
||||||
fun delete(id: String)
|
fun delete(id: String)
|
||||||
|
fun prune(olderThanTimestampMillis: Long, maxCount: Int)
|
||||||
}
|
}
|
||||||
|
|
||||||
expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore
|
expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore
|
||||||
@@ -15,14 +16,17 @@ expect fun createPendingCrashStore(appDataDir: String): PendingCrashStore
|
|||||||
internal object CrashReportCodec {
|
internal object CrashReportCodec {
|
||||||
private const val FieldSep = "\u001f"
|
private const val FieldSep = "\u001f"
|
||||||
private const val RecordSep = "\u001e"
|
private const val RecordSep = "\u001e"
|
||||||
|
private const val Version2Prefix = "vnidrop-crash-v2\n"
|
||||||
|
private const val MaxEncodedChars = 512 * 1024
|
||||||
|
|
||||||
fun encode(report: CrashReport): String = buildString {
|
fun encode(report: CrashReport): String = buildString {
|
||||||
fun field(key: String, value: String) {
|
fun field(key: String, value: String) {
|
||||||
append(key)
|
append(key)
|
||||||
append('=')
|
append('=')
|
||||||
append(value.replace("\n", "\\n").replace("\r", "\\r"))
|
append(value.hexEncode())
|
||||||
append(FieldSep)
|
append('\n')
|
||||||
}
|
}
|
||||||
|
append(Version2Prefix)
|
||||||
field("id", report.id)
|
field("id", report.id)
|
||||||
field("ts", report.timestampMillis.toString())
|
field("ts", report.timestampMillis.toString())
|
||||||
field("install", report.installId)
|
field("install", report.installId)
|
||||||
@@ -31,22 +35,70 @@ internal object CrashReportCodec {
|
|||||||
field("type", report.exceptionType)
|
field("type", report.exceptionType)
|
||||||
field("message", report.exceptionMessage)
|
field("message", report.exceptionMessage)
|
||||||
field("stack", report.stackTrace)
|
field("stack", report.stackTrace)
|
||||||
field("diag", if (report.diagnosticsEnabledAtCapture) "1" else "0")
|
|
||||||
field("schema", report.schemaVersion.toString())
|
|
||||||
field(
|
field(
|
||||||
"crumbs",
|
"diag",
|
||||||
report.breadcrumbs.joinToString(RecordSep) { crumb ->
|
when (report.diagnosticsEnabledAtCapture) {
|
||||||
listOf(
|
true -> "1"
|
||||||
crumb.timestampMillis.toString(),
|
false -> "0"
|
||||||
crumb.name,
|
null -> "u"
|
||||||
crumb.properties.entries.joinToString(",") { "${it.key}:${it.value}" },
|
|
||||||
).joinToString("|")
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
field("schema", report.schemaVersion.toString())
|
||||||
|
val breadcrumbs = report.breadcrumbs.take(40)
|
||||||
|
field("crumb.count", breadcrumbs.size.toString())
|
||||||
|
breadcrumbs.forEachIndexed { crumbIndex, crumb ->
|
||||||
|
field("crumb.$crumbIndex.ts", crumb.timestampMillis.toString())
|
||||||
|
field("crumb.$crumbIndex.name", crumb.name)
|
||||||
|
val properties = crumb.properties.entries.take(MaxDiagnosticProperties)
|
||||||
|
field("crumb.$crumbIndex.prop.count", properties.size.toString())
|
||||||
|
properties.forEachIndexed { propertyIndex, (key, value) ->
|
||||||
|
field("crumb.$crumbIndex.prop.$propertyIndex.key", key)
|
||||||
|
field("crumb.$crumbIndex.prop.$propertyIndex.value", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun decode(raw: String): CrashReport? {
|
fun decode(raw: String): CrashReport? {
|
||||||
if (raw.isBlank()) return null
|
if (raw.isBlank() || raw.length > MaxEncodedChars) return null
|
||||||
|
return if (raw.startsWith(Version2Prefix)) decodeVersion2(raw) else decodeLegacy(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun decodeVersion2(raw: String): CrashReport? {
|
||||||
|
val map = linkedMapOf<String, String>()
|
||||||
|
for (part in raw.removePrefix(Version2Prefix).lineSequence()) {
|
||||||
|
if (part.isEmpty()) continue
|
||||||
|
val eq = part.indexOf('=')
|
||||||
|
if (eq <= 0) continue
|
||||||
|
val key = part.substring(0, eq)
|
||||||
|
val value = part.substring(eq + 1).hexDecode() ?: return null
|
||||||
|
map[key] = value
|
||||||
|
}
|
||||||
|
val crumbCount = map["crumb.count"]?.toIntOrNull()?.takeIf { it in 0..40 } ?: return null
|
||||||
|
val crumbs = buildList {
|
||||||
|
repeat(crumbCount) { crumbIndex ->
|
||||||
|
val timestamp = map["crumb.$crumbIndex.ts"]
|
||||||
|
?.toLongOrNull()
|
||||||
|
?.takeIf { it >= 0 }
|
||||||
|
?: return null
|
||||||
|
val name = map["crumb.$crumbIndex.name"]?.takeIf { it.isNotBlank() } ?: return null
|
||||||
|
val propertyCount = map["crumb.$crumbIndex.prop.count"]
|
||||||
|
?.toIntOrNull()
|
||||||
|
?.takeIf { it in 0..MaxDiagnosticProperties }
|
||||||
|
?: return null
|
||||||
|
val properties = buildMap {
|
||||||
|
repeat(propertyCount) { propertyIndex ->
|
||||||
|
val key = map["crumb.$crumbIndex.prop.$propertyIndex.key"] ?: return null
|
||||||
|
val value = map["crumb.$crumbIndex.prop.$propertyIndex.value"] ?: return null
|
||||||
|
put(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
add(Breadcrumb(name = name, timestampMillis = timestamp, properties = properties))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return reportFromFields(map, crumbs)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun decodeLegacy(raw: String): CrashReport? {
|
||||||
val map = linkedMapOf<String, String>()
|
val map = linkedMapOf<String, String>()
|
||||||
for (part in raw.split(FieldSep)) {
|
for (part in raw.split(FieldSep)) {
|
||||||
if (part.isEmpty()) continue
|
if (part.isEmpty()) continue
|
||||||
@@ -58,15 +110,14 @@ internal object CrashReportCodec {
|
|||||||
.replace("\\r", "\r")
|
.replace("\\r", "\r")
|
||||||
map[key] = value
|
map[key] = value
|
||||||
}
|
}
|
||||||
val id = map["id"] ?: return null
|
|
||||||
val crumbs = map["crumbs"].orEmpty()
|
val crumbs = map["crumbs"].orEmpty()
|
||||||
.split(RecordSep)
|
.split(RecordSep)
|
||||||
.filter { it.isNotBlank() }
|
.filter { it.isNotBlank() }
|
||||||
.mapNotNull { entry ->
|
.mapNotNull { entry ->
|
||||||
val pieces = entry.split('|', limit = 3)
|
val pieces = entry.split('|', limit = 3)
|
||||||
if (pieces.size < 2) return@mapNotNull null
|
if (pieces.size < 2) return@mapNotNull null
|
||||||
val ts = pieces[0].toLongOrNull() ?: return@mapNotNull null
|
val ts = pieces[0].toLongOrNull()?.takeIf { it >= 0 } ?: return@mapNotNull null
|
||||||
val name = pieces[1]
|
val name = pieces[1].takeIf { it.isNotBlank() } ?: return@mapNotNull null
|
||||||
val props = if (pieces.size > 2 && pieces[2].isNotBlank()) {
|
val props = if (pieces.size > 2 && pieces[2].isNotBlank()) {
|
||||||
pieces[2].split(',').mapNotNull { kv ->
|
pieces[2].split(',').mapNotNull { kv ->
|
||||||
val colon = kv.indexOf(':')
|
val colon = kv.indexOf(':')
|
||||||
@@ -78,18 +129,65 @@ internal object CrashReportCodec {
|
|||||||
}
|
}
|
||||||
Breadcrumb(name = name, timestampMillis = ts, properties = props)
|
Breadcrumb(name = name, timestampMillis = ts, properties = props)
|
||||||
}
|
}
|
||||||
|
return reportFromFields(map, crumbs)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun reportFromFields(
|
||||||
|
map: Map<String, String>,
|
||||||
|
crumbs: List<Breadcrumb>,
|
||||||
|
): CrashReport? {
|
||||||
|
val id = map["id"]?.takeIf(::isValidDiagnosticId) ?: return null
|
||||||
|
val timestamp = map["ts"]?.toLongOrNull()?.takeIf { it >= 0 } ?: return null
|
||||||
|
val exceptionType = map["type"]?.takeIf { it.isNotBlank() } ?: return null
|
||||||
|
val schemaVersion = map["schema"]?.toIntOrNull()
|
||||||
|
?.takeIf { it == DiagnosticsSchemaVersion }
|
||||||
|
?: return null
|
||||||
|
val diagnosticsEnabled: Boolean? = when (map["diag"]) {
|
||||||
|
"1" -> true
|
||||||
|
"0" -> false
|
||||||
|
"u" -> null
|
||||||
|
else -> return null
|
||||||
|
}
|
||||||
return CrashReport(
|
return CrashReport(
|
||||||
id = id,
|
id = id,
|
||||||
timestampMillis = map["ts"]?.toLongOrNull() ?: 0L,
|
timestampMillis = timestamp,
|
||||||
installId = map["install"].orEmpty(),
|
installId = map["install"].orEmpty(),
|
||||||
appVersion = map["app"].orEmpty(),
|
appVersion = map["app"].orEmpty(),
|
||||||
platform = map["platform"].orEmpty(),
|
platform = map["platform"].orEmpty(),
|
||||||
exceptionType = map["type"].orEmpty(),
|
exceptionType = exceptionType,
|
||||||
exceptionMessage = map["message"].orEmpty(),
|
exceptionMessage = map["message"].orEmpty(),
|
||||||
stackTrace = map["stack"].orEmpty(),
|
stackTrace = map["stack"].orEmpty(),
|
||||||
breadcrumbs = crumbs,
|
breadcrumbs = crumbs,
|
||||||
diagnosticsEnabledAtCapture = map["diag"] == "1",
|
diagnosticsEnabledAtCapture = diagnosticsEnabled,
|
||||||
schemaVersion = map["schema"]?.toIntOrNull() ?: DiagnosticsSchemaVersion,
|
schemaVersion = schemaVersion,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal fun isValidDiagnosticId(id: String): Boolean =
|
||||||
|
DiagnosticIdPattern.matches(id)
|
||||||
|
|
||||||
|
private val DiagnosticIdPattern =
|
||||||
|
Regex("^[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$")
|
||||||
|
|
||||||
|
private fun String.hexEncode(): String {
|
||||||
|
val digits = "0123456789abcdef"
|
||||||
|
return buildString(length * 2) {
|
||||||
|
for (byte in this@hexEncode.encodeToByteArray()) {
|
||||||
|
val value = byte.toInt() and 0xff
|
||||||
|
append(digits[value ushr 4])
|
||||||
|
append(digits[value and 0x0f])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.hexDecode(): String? {
|
||||||
|
if (length % 2 != 0) return null
|
||||||
|
val bytes = ByteArray(length / 2)
|
||||||
|
for (index in bytes.indices) {
|
||||||
|
val high = this[index * 2].digitToIntOrNull(16) ?: return null
|
||||||
|
val low = this[index * 2 + 1].digitToIntOrNull(16) ?: return null
|
||||||
|
bytes[index] = ((high shl 4) or low).toByte()
|
||||||
|
}
|
||||||
|
return runCatching { bytes.decodeToString(throwOnInvalidSequence = true) }.getOrNull()
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
data class PlatformHttpResponse(
|
||||||
|
val statusCode: Int,
|
||||||
|
val body: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect suspend fun platformHttpPost(
|
||||||
|
url: String,
|
||||||
|
headers: Map<String, String>,
|
||||||
|
bodyUtf8: String,
|
||||||
|
): PlatformHttpResponse
|
||||||
@@ -2,17 +2,25 @@ package com.vnidrop.app.diagnostics
|
|||||||
|
|
||||||
import com.vnidrop.app.logging.platformNowMillis
|
import com.vnidrop.app.logging.platformNowMillis
|
||||||
import com.vnidrop.app.preferences.PreferencesRepository
|
import com.vnidrop.app.preferences.PreferencesRepository
|
||||||
|
import com.vnidrop.app.util.randomUuidString
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
|
import kotlinx.coroutines.channels.Channel
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.flow.distinctUntilChanged
|
import kotlinx.coroutines.flow.distinctUntilChanged
|
||||||
import kotlinx.coroutines.flow.map
|
import kotlinx.coroutines.flow.map
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.sync.Mutex
|
import kotlinx.coroutines.sync.Mutex
|
||||||
import kotlinx.coroutines.sync.withLock
|
import kotlinx.coroutines.sync.withLock
|
||||||
|
import kotlinx.coroutines.withTimeoutOrNull
|
||||||
|
import kotlin.concurrent.atomics.AtomicReference
|
||||||
|
import kotlin.concurrent.atomics.ExperimentalAtomicApi
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Product telemetry: sparse events, gated by diagnostics opt-in.
|
* Product telemetry: sparse events, gated by diagnostics opt-in.
|
||||||
* Events are buffered and flushed in batches when transport is available.
|
* Events are buffered and flushed in batches when transport is available.
|
||||||
*/
|
*/
|
||||||
|
@OptIn(ExperimentalAtomicApi::class)
|
||||||
class TelemetryRecorder(
|
class TelemetryRecorder(
|
||||||
private val preferencesRepository: PreferencesRepository,
|
private val preferencesRepository: PreferencesRepository,
|
||||||
private val transport: DiagnosticsTransport,
|
private val transport: DiagnosticsTransport,
|
||||||
@@ -20,64 +28,186 @@ class TelemetryRecorder(
|
|||||||
private val scope: CoroutineScope,
|
private val scope: CoroutineScope,
|
||||||
private val maxBufferSize: Int = DefaultMaxBuffer,
|
private val maxBufferSize: Int = DefaultMaxBuffer,
|
||||||
private val flushThreshold: Int = DefaultFlushThreshold,
|
private val flushThreshold: Int = DefaultFlushThreshold,
|
||||||
|
private val flushIntervalMillis: Long = DefaultFlushIntervalMillis,
|
||||||
|
private val retryBackoffMillis: Long = DefaultRetryBackoffMillis,
|
||||||
|
private val automaticRetryCount: Int = DefaultAutomaticRetryCount,
|
||||||
) {
|
) {
|
||||||
private val bufferMutex = Mutex()
|
private val bufferMutex = Mutex()
|
||||||
@Volatile private var buffer: List<TelemetryEvent> = emptyList()
|
private val state = AtomicReference(TelemetryState())
|
||||||
@Volatile private var enabled: Boolean = false
|
private val flushSignals = Channel<Unit>(Channel.CONFLATED)
|
||||||
|
|
||||||
init {
|
init {
|
||||||
|
require(maxBufferSize > 0) { "maxBufferSize must be positive" }
|
||||||
|
require(flushThreshold > 0) { "flushThreshold must be positive" }
|
||||||
|
require(flushIntervalMillis > 0) { "flushIntervalMillis must be positive" }
|
||||||
|
require(retryBackoffMillis > 0) { "retryBackoffMillis must be positive" }
|
||||||
|
require(automaticRetryCount >= 0) { "automaticRetryCount must not be negative" }
|
||||||
scope.launch {
|
scope.launch {
|
||||||
preferencesRepository.preferences
|
preferencesRepository.preferences
|
||||||
.map { it.diagnosticsEnabled }
|
.map { it.diagnosticsEnabled }
|
||||||
.distinctUntilChanged()
|
.distinctUntilChanged()
|
||||||
.collect { isEnabled ->
|
.collect { isEnabled ->
|
||||||
enabled = isEnabled
|
updateState { current ->
|
||||||
if (!isEnabled) {
|
if (isEnabled) current.copy(enabled = true) else TelemetryState(enabled = false)
|
||||||
buffer = emptyList()
|
}
|
||||||
}
|
flushSignals.trySend(Unit)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
scope.launch { runAutomaticFlushes() }
|
||||||
}
|
}
|
||||||
|
|
||||||
fun record(name: String, properties: Map<String, String> = emptyMap()) {
|
fun record(name: String, properties: Map<String, String> = emptyMap()) {
|
||||||
if (!DiagnosticsBuildConfig.INCLUDED) return
|
if (!DiagnosticsBuildConfig.INCLUDED) return
|
||||||
val redacted = LogRedactor.redactMap(properties)
|
val sanitizedName = sanitizeDiagnosticName(name)
|
||||||
breadcrumbs.add(name, redacted)
|
if (sanitizedName.isBlank()) return
|
||||||
if (!enabled) return
|
val sanitizedProperties = sanitizeDiagnosticProperties(properties)
|
||||||
|
breadcrumbs.add(sanitizedName, sanitizedProperties)
|
||||||
val event = TelemetryEvent(
|
val event = TelemetryEvent(
|
||||||
name = name.take(MaxNameLength),
|
name = sanitizedName,
|
||||||
timestampMillis = platformNowMillis(),
|
timestampMillis = platformNowMillis(),
|
||||||
properties = redacted.mapValues { it.value.take(MaxPropertyValueLength) },
|
properties = sanitizedProperties,
|
||||||
)
|
)
|
||||||
val next = (buffer + event).takeLast(maxBufferSize)
|
while (true) {
|
||||||
buffer = next
|
val current = state.load()
|
||||||
if (next.size >= flushThreshold) {
|
if (current.enabled == false) return
|
||||||
scope.launch { flush() }
|
val remainingCapacity =
|
||||||
|
(maxBufferSize - current.retryBatch?.events.orEmpty().size).coerceAtLeast(0)
|
||||||
|
val nextBuffer = (current.buffer + event).takeLast(remainingCapacity)
|
||||||
|
if (state.compareAndSet(current, current.copy(buffer = nextBuffer))) {
|
||||||
|
flushSignals.trySend(Unit)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
suspend fun flush(): Result<Unit> = bufferMutex.withLock {
|
suspend fun flush(): Result<Unit> {
|
||||||
if (!enabled) {
|
return bufferMutex.withLock {
|
||||||
buffer = emptyList()
|
var discardedFailure: Throwable? = null
|
||||||
return Result.success(Unit)
|
var outcome: Result<Unit>? = null
|
||||||
|
while (outcome == null) {
|
||||||
|
val current = state.load()
|
||||||
|
if (current.enabled != true) return@withLock Result.success(Unit)
|
||||||
|
val pendingRetry = current.retryBatch
|
||||||
|
val events = if (pendingRetry == null) nextBatchEvents(current.buffer) else emptyList()
|
||||||
|
if (pendingRetry == null && events.isEmpty()) {
|
||||||
|
outcome = discardedFailure?.let { Result.failure(it) } ?: Result.success(Unit)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
val batch = buffer
|
val batch: TelemetryBatch
|
||||||
if (batch.isEmpty()) return Result.success(Unit)
|
if (pendingRetry != null) {
|
||||||
buffer = emptyList()
|
batch = pendingRetry
|
||||||
val result = transport.sendEvents(batch)
|
} else {
|
||||||
if (result.isFailure) {
|
val prepared = TelemetryBatch(id = randomUuidString(), events = events)
|
||||||
// Re-queue on failure so a future transport can retry once online.
|
val next = current.copy(
|
||||||
buffer = (batch + buffer).takeLast(maxBufferSize)
|
buffer = current.buffer.drop(events.size),
|
||||||
|
retryBatch = prepared,
|
||||||
|
)
|
||||||
|
if (!state.compareAndSet(current, next)) continue
|
||||||
|
batch = prepared
|
||||||
|
}
|
||||||
|
if (state.load().retryBatch != batch) continue
|
||||||
|
val result = try {
|
||||||
|
transport.sendEvents(batch)
|
||||||
|
} catch (cancelled: CancellationException) {
|
||||||
|
throw cancelled
|
||||||
|
} catch (error: Throwable) {
|
||||||
|
Result.failure(error)
|
||||||
|
}
|
||||||
|
if (result.isSuccess) {
|
||||||
|
clearRetryBatch(batch)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val error = result.exceptionOrNull() ?: IllegalStateException("diagnostics event delivery failed")
|
||||||
|
if (error.isPermanentDiagnosticsPayloadRejection()) {
|
||||||
|
clearRetryBatch(batch)
|
||||||
|
discardedFailure = discardedFailure ?: error
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
outcome = result
|
||||||
|
}
|
||||||
|
checkNotNull(outcome)
|
||||||
}
|
}
|
||||||
return result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fun pendingCount(): Int = buffer.size
|
private fun nextBatchEvents(events: List<TelemetryEvent>): List<TelemetryEvent> {
|
||||||
|
if (events.isEmpty()) return emptyList()
|
||||||
|
var minimum = 1
|
||||||
|
var maximum = minOf(events.size, MaxEventsPerBatch)
|
||||||
|
var accepted = 1
|
||||||
|
while (minimum <= maximum) {
|
||||||
|
val candidateSize = minimum + (maximum - minimum) / 2
|
||||||
|
if (DiagnosticsJson.eventBatchFitsRequest(events.take(candidateSize))) {
|
||||||
|
accepted = candidateSize
|
||||||
|
minimum = candidateSize + 1
|
||||||
|
} else {
|
||||||
|
maximum = candidateSize - 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return events.take(accepted)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun pendingCount(): Int {
|
||||||
|
val current = state.load()
|
||||||
|
return current.retryBatch?.events.orEmpty().size + current.buffer.size
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun runAutomaticFlushes() {
|
||||||
|
while (true) {
|
||||||
|
flushSignals.receive()
|
||||||
|
var retries = 0
|
||||||
|
while (true) {
|
||||||
|
val current = state.load()
|
||||||
|
if (current.enabled != true || pendingCount() == 0) break
|
||||||
|
if (current.retryBatch == null && pendingCount() < flushThreshold) {
|
||||||
|
val signalled = withTimeoutOrNull(flushIntervalMillis) {
|
||||||
|
flushSignals.receive()
|
||||||
|
true
|
||||||
|
} ?: false
|
||||||
|
if (signalled) continue
|
||||||
|
}
|
||||||
|
|
||||||
|
val result = flush()
|
||||||
|
if (result.isSuccess || pendingCount() == 0) {
|
||||||
|
retries = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val error = result.exceptionOrNull()
|
||||||
|
if (error?.isPermanentDiagnosticsPayloadRejection() == true || retries >= automaticRetryCount) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
retries += 1
|
||||||
|
delay(retryBackoffMillis)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun clearRetryBatch(batch: TelemetryBatch) {
|
||||||
|
while (true) {
|
||||||
|
val current = state.load()
|
||||||
|
if (current.retryBatch != batch) return
|
||||||
|
if (state.compareAndSet(current, current.copy(retryBatch = null))) return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun updateState(update: (TelemetryState) -> TelemetryState) {
|
||||||
|
while (true) {
|
||||||
|
val current = state.load()
|
||||||
|
if (state.compareAndSet(current, update(current))) return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
const val DefaultMaxBuffer = 100
|
const val DefaultMaxBuffer = 100
|
||||||
const val DefaultFlushThreshold = 20
|
const val DefaultFlushThreshold = 20
|
||||||
private const val MaxNameLength = 64
|
const val MaxEventsPerBatch = 50
|
||||||
private const val MaxPropertyValueLength = 128
|
const val DefaultFlushIntervalMillis = 30_000L
|
||||||
|
const val DefaultRetryBackoffMillis = 30_000L
|
||||||
|
const val DefaultAutomaticRetryCount = 3
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private data class TelemetryState(
|
||||||
|
val enabled: Boolean? = null,
|
||||||
|
val buffer: List<TelemetryEvent> = emptyList(),
|
||||||
|
val retryBatch: TelemetryBatch? = null,
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,27 +1,250 @@
|
|||||||
package com.vnidrop.app.diagnostics
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
import com.vnidrop.app.DeviceInfo
|
import com.vnidrop.app.DeviceInfo
|
||||||
import com.vnidrop.app.preferences.AppPreferences
|
|
||||||
import com.vnidrop.app.core.ReceiveFolder
|
import com.vnidrop.app.core.ReceiveFolder
|
||||||
import com.vnidrop.app.core.ReceiveFolderKind
|
import com.vnidrop.app.core.ReceiveFolderKind
|
||||||
|
import com.vnidrop.app.preferences.AppPreferences
|
||||||
|
import com.vnidrop.app.preferences.PreferencesRepository
|
||||||
import com.vnidrop.app.support.FakePreferencesRepository
|
import com.vnidrop.app.support.FakePreferencesRepository
|
||||||
import com.vnidrop.app.ui.theme.ThemeMode
|
import com.vnidrop.app.ui.theme.ThemeMode
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
|
import kotlinx.coroutines.CompletableDeferred
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||||
|
import kotlinx.coroutines.coroutineScope
|
||||||
|
import kotlinx.coroutines.flow.emitAll
|
||||||
|
import kotlinx.coroutines.flow.flow
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.test.TestScope
|
import kotlinx.coroutines.test.TestScope
|
||||||
import kotlinx.coroutines.test.UnconfinedTestDispatcher
|
import kotlinx.coroutines.test.UnconfinedTestDispatcher
|
||||||
|
import kotlinx.coroutines.test.advanceTimeBy
|
||||||
import kotlinx.coroutines.test.advanceUntilIdle
|
import kotlinx.coroutines.test.advanceUntilIdle
|
||||||
|
import kotlinx.coroutines.test.runCurrent
|
||||||
import kotlinx.coroutines.test.runTest
|
import kotlinx.coroutines.test.runTest
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
import kotlin.test.assertFalse
|
import kotlin.test.assertFalse
|
||||||
|
import kotlin.test.assertFailsWith
|
||||||
|
import kotlin.test.assertIs
|
||||||
|
import kotlin.test.assertNull
|
||||||
import kotlin.test.assertTrue
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
@OptIn(ExperimentalCoroutinesApi::class)
|
@OptIn(ExperimentalCoroutinesApi::class)
|
||||||
class DiagnosticsTest {
|
class DiagnosticsTest {
|
||||||
@Test
|
@Test
|
||||||
fun diagnosticsBuildConfigDefaultsToIncluded() {
|
fun diagnosticsBuildConfigDefaultsToIncludedWithEmptyEndpoint() {
|
||||||
// Production default is true; builds can override with -Pvnidrop.diagnostics.included=false.
|
// Production default is true; builds can override with -Pvnidrop.diagnostics.included=false.
|
||||||
assertTrue(DiagnosticsBuildConfig.INCLUDED)
|
assertTrue(DiagnosticsBuildConfig.INCLUDED)
|
||||||
|
// Empty endpoint keeps shipping safe (NoOp transport) until Cloudflare is configured.
|
||||||
|
assertEquals("", DiagnosticsBuildConfig.ENDPOINT)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun diagnosticsJsonEscapesAndShapesPayloads() {
|
||||||
|
val eventsJson = DiagnosticsJson.eventsBody(
|
||||||
|
batchId = "batch-1",
|
||||||
|
installId = "inst-1",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
events = listOf(
|
||||||
|
TelemetryEvent("app_open", 10L, mapOf("a" to "quote\"here")),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertTrue(eventsJson.contains("\"batchId\":\"batch-1\""))
|
||||||
|
assertTrue(eventsJson.contains("\"installId\":\"inst-1\""))
|
||||||
|
assertTrue(eventsJson.contains("\"name\":\"app_open\""))
|
||||||
|
assertTrue(eventsJson.contains("quote\\\"here"))
|
||||||
|
|
||||||
|
val crashJson = DiagnosticsJson.crashBody(
|
||||||
|
CrashReport(
|
||||||
|
id = "c1",
|
||||||
|
timestampMillis = 1L,
|
||||||
|
installId = "inst",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
exceptionType = "E",
|
||||||
|
exceptionMessage = "line\nbreak",
|
||||||
|
stackTrace = "stack",
|
||||||
|
breadcrumbs = emptyList(),
|
||||||
|
diagnosticsEnabledAtCapture = true,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertTrue(crashJson.contains("line\\nbreak"))
|
||||||
|
assertTrue(crashJson.contains("\"diagnosticsEnabledAtCapture\":true"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun diagnosticsJsonKeepsEscapedBugPayloadWithinWorkerLimit() {
|
||||||
|
val report = BugReport(
|
||||||
|
id = "b",
|
||||||
|
timestampMillis = 1L,
|
||||||
|
installId = "i",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
whatHappened = "w",
|
||||||
|
expected = "e",
|
||||||
|
steps = "",
|
||||||
|
contact = "",
|
||||||
|
includeLogs = true,
|
||||||
|
logs = "\n".repeat(BugReportService.MaxLogBytes),
|
||||||
|
device = DeviceSnapshot(null, null, "OS", null, null),
|
||||||
|
breadcrumbs = emptyList(),
|
||||||
|
)
|
||||||
|
|
||||||
|
val body = DiagnosticsJson.bugBody(report)
|
||||||
|
|
||||||
|
assertTrue(body.encodeToByteArray().size <= DiagnosticsJson.MaxRequestBytes)
|
||||||
|
assertTrue(body.contains("\"includeLogs\":true"))
|
||||||
|
assertTrue(body.endsWith("}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun httpTransportPostsExpectedPaths() = runTest {
|
||||||
|
val calls = mutableListOf<Pair<String, String>>()
|
||||||
|
val acknowledgementIds = ArrayDeque(listOf("batch-1", "c", "b"))
|
||||||
|
val transport = HttpDiagnosticsTransport(
|
||||||
|
baseUrl = "https://diag.example",
|
||||||
|
ingestKey = "secret",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
installIdProvider = { "install-x" },
|
||||||
|
post = { url, headers, body ->
|
||||||
|
assertEquals("secret", headers["X-VniDrop-Key"])
|
||||||
|
calls += url to body
|
||||||
|
PlatformHttpResponse(
|
||||||
|
202,
|
||||||
|
"""{"ok":true,"id":"${acknowledgementIds.removeFirst()}","stored":1}""",
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
val eventResult = transport.sendEvents(
|
||||||
|
TelemetryBatch("batch-1", listOf(TelemetryEvent("nav", 1L))),
|
||||||
|
)
|
||||||
|
assertTrue(eventResult.isSuccess)
|
||||||
|
assertEquals("https://diag.example/v1/events", calls[0].first)
|
||||||
|
assertTrue(calls[0].second.contains("install-x"))
|
||||||
|
|
||||||
|
val crashResult = transport.sendCrash(
|
||||||
|
CrashReport(
|
||||||
|
id = "c",
|
||||||
|
timestampMillis = 1L,
|
||||||
|
installId = "i",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
exceptionType = "E",
|
||||||
|
exceptionMessage = "m",
|
||||||
|
stackTrace = "s",
|
||||||
|
breadcrumbs = emptyList(),
|
||||||
|
diagnosticsEnabledAtCapture = true,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertTrue(crashResult.isSuccess)
|
||||||
|
assertEquals("https://diag.example/v1/crashes", calls[1].first)
|
||||||
|
|
||||||
|
val bugResult = transport.sendBugReport(
|
||||||
|
BugReport(
|
||||||
|
id = "b",
|
||||||
|
timestampMillis = 1L,
|
||||||
|
installId = "i",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
whatHappened = "w",
|
||||||
|
expected = "e",
|
||||||
|
steps = "",
|
||||||
|
contact = "",
|
||||||
|
includeLogs = false,
|
||||||
|
logs = "",
|
||||||
|
device = DeviceSnapshot(null, null, "OS", null, null),
|
||||||
|
breadcrumbs = emptyList(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertTrue(bugResult.isSuccess)
|
||||||
|
assertEquals("https://diag.example/v1/bugs", calls[2].first)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun httpTransportFailsOnHttpError() = runTest {
|
||||||
|
val transport = HttpDiagnosticsTransport(
|
||||||
|
baseUrl = "https://diag.example",
|
||||||
|
ingestKey = "secret",
|
||||||
|
post = { _, _, _ -> PlatformHttpResponse(401, """{"error":"unauthorized"}""") },
|
||||||
|
)
|
||||||
|
assertTrue(
|
||||||
|
transport.sendEvents(TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1L)))).isFailure,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun httpTransportPropagatesCancellation() = runTest {
|
||||||
|
val transport = HttpDiagnosticsTransport(
|
||||||
|
baseUrl = "https://diag.example",
|
||||||
|
ingestKey = "secret",
|
||||||
|
post = { _, _, _ -> throw CancellationException("cancelled") },
|
||||||
|
)
|
||||||
|
|
||||||
|
assertFailsWith<CancellationException> {
|
||||||
|
transport.sendEvents(TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun httpTransportRejectsMissingOrNegativeAcknowledgement() = runTest {
|
||||||
|
val responses = ArrayDeque(
|
||||||
|
listOf(
|
||||||
|
PlatformHttpResponse(202, ""),
|
||||||
|
PlatformHttpResponse(202, """{"ok":false}"""),
|
||||||
|
PlatformHttpResponse(202, """{,"ok":true}"""),
|
||||||
|
PlatformHttpResponse(202, """{"ok":true,"id":"different"}"""),
|
||||||
|
PlatformHttpResponse(202, """{"ok":true,"id":"batch-4","id":"batch-4"}"""),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
val transport = HttpDiagnosticsTransport(
|
||||||
|
baseUrl = "https://diag.example",
|
||||||
|
ingestKey = "secret",
|
||||||
|
post = { _, _, _ -> responses.removeFirst() },
|
||||||
|
)
|
||||||
|
|
||||||
|
repeat(5) { index ->
|
||||||
|
val result = transport.sendEvents(
|
||||||
|
TelemetryBatch("batch-$index", listOf(TelemetryEvent("x", 1L))),
|
||||||
|
)
|
||||||
|
assertIs<DiagnosticsProtocolException>(result.exceptionOrNull())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun httpTransportRequiresHttpsExceptForLoopbackDevelopment() {
|
||||||
|
assertFailsWith<IllegalArgumentException> {
|
||||||
|
HttpDiagnosticsTransport("http://diag.example", "secret")
|
||||||
|
}
|
||||||
|
assertFailsWith<IllegalArgumentException> {
|
||||||
|
HttpDiagnosticsTransport("http://[::1].example", "secret")
|
||||||
|
}
|
||||||
|
HttpDiagnosticsTransport("http://localhost:8787", "secret")
|
||||||
|
HttpDiagnosticsTransport("http://127.0.0.1:8787", "secret")
|
||||||
|
HttpDiagnosticsTransport("http://[::1]:8787", "secret")
|
||||||
|
assertFailsWith<IllegalArgumentException> {
|
||||||
|
HttpDiagnosticsTransport("https://diag.example", "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun createTransportIsNoOpWhenEndpointBlank() {
|
||||||
|
// With default generated config endpoint is empty.
|
||||||
|
val transport = createDiagnosticsTransport(
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
installIdProvider = { "id" },
|
||||||
|
)
|
||||||
|
assertIs<NoOpDiagnosticsTransport>(transport)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun noOpTransportReportsUnavailableDelivery() = runTest {
|
||||||
|
val result = NoOpDiagnosticsTransport().sendEvents(
|
||||||
|
TelemetryBatch("batch-1", listOf(TelemetryEvent("x", 1L))),
|
||||||
|
)
|
||||||
|
assertIs<DiagnosticsUnavailableException>(result.exceptionOrNull())
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -72,6 +295,35 @@ class DiagnosticsTest {
|
|||||||
assertEquals(1, breadcrumbs.snapshot().size)
|
assertEquals(1, breadcrumbs.snapshot().size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryRetainsColdStartEventsUntilConsentLoads() = runTest {
|
||||||
|
val backing = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val preferenceGate = CompletableDeferred<Unit>()
|
||||||
|
val delayedPreferences = object : PreferencesRepository by backing {
|
||||||
|
override val preferences = flow {
|
||||||
|
preferenceGate.await()
|
||||||
|
emitAll(backing.preferences)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = delayedPreferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 20,
|
||||||
|
)
|
||||||
|
runCurrent()
|
||||||
|
|
||||||
|
recorder.record("app_open")
|
||||||
|
assertEquals(1, recorder.pendingCount())
|
||||||
|
preferenceGate.complete(Unit)
|
||||||
|
runCurrent()
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
|
||||||
|
assertEquals(listOf("app_open"), transport.events.single().map { it.name })
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun telemetryBuffersAndFlushesWhenEnabled() = runTest {
|
fun telemetryBuffersAndFlushesWhenEnabled() = runTest {
|
||||||
val preferences = fakePrefs(diagnosticsEnabled = true)
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
@@ -92,6 +344,222 @@ class DiagnosticsTest {
|
|||||||
assertEquals(listOf("one", "two"), transport.events.single().map { it.name })
|
assertEquals(listOf("one", "two"), transport.events.single().map { it.name })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryFlushesSparseEventsAfterTheInterval() = runTest {
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 20,
|
||||||
|
flushIntervalMillis = 1_000,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
recorder.record("sparse")
|
||||||
|
|
||||||
|
advanceTimeBy(999)
|
||||||
|
runCurrent()
|
||||||
|
assertTrue(transport.eventBatches.isEmpty())
|
||||||
|
advanceTimeBy(1)
|
||||||
|
runCurrent()
|
||||||
|
assertEquals(listOf("sparse"), transport.events.single().map { it.name })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryCoalescesAutomaticRetriesWithBackoff() = runTest {
|
||||||
|
val transport = RecordingDiagnosticsTransport().apply {
|
||||||
|
eventsResult = Result.failure(IllegalStateException("offline"))
|
||||||
|
}
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 1,
|
||||||
|
flushIntervalMillis = 10_000,
|
||||||
|
retryBackoffMillis = 1_000,
|
||||||
|
automaticRetryCount = 1,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
recorder.record("retry")
|
||||||
|
runCurrent()
|
||||||
|
assertEquals(1, transport.eventBatches.size)
|
||||||
|
|
||||||
|
advanceTimeBy(999)
|
||||||
|
runCurrent()
|
||||||
|
assertEquals(1, transport.eventBatches.size)
|
||||||
|
advanceTimeBy(1)
|
||||||
|
runCurrent()
|
||||||
|
assertEquals(2, transport.eventBatches.size)
|
||||||
|
advanceTimeBy(10_000)
|
||||||
|
runCurrent()
|
||||||
|
assertEquals(2, transport.eventBatches.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryFlushesAtMostFiftyEventsPerBatch() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 101,
|
||||||
|
maxBufferSize = 100,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
repeat(75) { recorder.record("event-$it") }
|
||||||
|
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
assertEquals(listOf(50, 25), transport.eventBatches.map { it.events.size })
|
||||||
|
assertTrue(transport.eventBatches.all { it.events.size <= TelemetryRecorder.MaxEventsPerBatch })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetrySplitsBatchesByEscapedRequestBytes() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val requestBodies = mutableListOf<String>()
|
||||||
|
val transport = HttpDiagnosticsTransport(
|
||||||
|
baseUrl = "https://diag.example",
|
||||||
|
ingestKey = "secret",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
installIdProvider = { "test-install" },
|
||||||
|
post = { _, _, body ->
|
||||||
|
requestBodies += body
|
||||||
|
val id = Regex(""""batchId":"([^"]+)"""").find(body)?.groupValues?.get(1)
|
||||||
|
PlatformHttpResponse(202, """{"ok":true,"id":"$id"}""")
|
||||||
|
},
|
||||||
|
)
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 101,
|
||||||
|
maxBufferSize = 100,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
val properties = LinkedHashMap<String, String>().apply {
|
||||||
|
repeat(MaxDiagnosticProperties) { index ->
|
||||||
|
put("key-$index-${"\u0001".repeat(40)}", "\u0001".repeat(MaxDiagnosticPropertyValueBytes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
repeat(50) { index ->
|
||||||
|
recorder.record("event-$index-${"\u0001".repeat(64)}", properties)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
assertTrue(requestBodies.size > 1)
|
||||||
|
assertTrue(requestBodies.all { it.encodeToByteArray().size <= DiagnosticsJson.MaxRequestBytes })
|
||||||
|
assertEquals(50, requestBodies.sumOf { body -> "\"schemaVersion\"".toRegex().findAll(body).count() })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetrySanitizesNamesAndPropertiesToServerByteLimits() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 100,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
val properties = LinkedHashMap<String, String>().apply {
|
||||||
|
repeat(20) { index -> put("key-$index-${"🙂".repeat(20)}", "🙂".repeat(100)) }
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder.record("🙂".repeat(100), properties)
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
|
||||||
|
val event = transport.eventBatches.single().events.single()
|
||||||
|
assertTrue(event.name.encodeToByteArray().size <= MaxDiagnosticNameBytes)
|
||||||
|
assertEquals(MaxDiagnosticProperties, event.properties.size)
|
||||||
|
assertTrue(event.properties.keys.all { it.encodeToByteArray().size <= MaxDiagnosticPropertyKeyBytes })
|
||||||
|
assertTrue(event.properties.values.all { it.encodeToByteArray().size <= MaxDiagnosticPropertyValueBytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryKeepsConcurrentRecordsWithoutExceedingItsBuffer() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 101,
|
||||||
|
maxBufferSize = 100,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
|
||||||
|
coroutineScope {
|
||||||
|
repeat(100) { index ->
|
||||||
|
launch(Dispatchers.Default) { recorder.record("event-$index") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
100,
|
||||||
|
recorder.pendingCount() + transport.eventBatches.sumOf { it.events.size },
|
||||||
|
)
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
assertEquals(100, transport.eventBatches.sumOf { it.events.size })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryRetryReusesBatchIdAndEvents() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val transport = RecordingDiagnosticsTransport().apply {
|
||||||
|
eventsResult = Result.failure(IllegalStateException("offline"))
|
||||||
|
}
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 10,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
recorder.record("one")
|
||||||
|
recorder.record("two")
|
||||||
|
|
||||||
|
assertTrue(recorder.flush().isFailure)
|
||||||
|
val firstAttempt = transport.eventBatches.single()
|
||||||
|
transport.eventsResult = Result.success(Unit)
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
|
||||||
|
assertEquals(listOf(firstAttempt, firstAttempt), transport.eventBatches)
|
||||||
|
assertEquals(0, recorder.pendingCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun telemetryDoesNotRequeuePermanentlyRejectedPayload() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val transport = RecordingDiagnosticsTransport().apply {
|
||||||
|
eventsResult = Result.failure(DiagnosticsHttpException(400, "/v1/events"))
|
||||||
|
}
|
||||||
|
val recorder = TelemetryRecorder(
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
flushThreshold = 10,
|
||||||
|
)
|
||||||
|
advanceUntilIdle()
|
||||||
|
recorder.record("invalid")
|
||||||
|
|
||||||
|
assertTrue(recorder.flush().isFailure)
|
||||||
|
assertEquals(0, recorder.pendingCount())
|
||||||
|
transport.eventsResult = Result.success(Unit)
|
||||||
|
assertTrue(recorder.flush().isSuccess)
|
||||||
|
assertEquals(1, transport.eventBatches.size)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun telemetryClearsBufferWhenOptedOut() = runTest {
|
fun telemetryClearsBufferWhenOptedOut() = runTest {
|
||||||
val preferences = fakePrefs(diagnosticsEnabled = true)
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
@@ -114,21 +582,49 @@ class DiagnosticsTest {
|
|||||||
@Test
|
@Test
|
||||||
fun crashCodecRoundTrips() {
|
fun crashCodecRoundTrips() {
|
||||||
val original = CrashReport(
|
val original = CrashReport(
|
||||||
id = "crash-1",
|
id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||||
timestampMillis = 42L,
|
timestampMillis = 42L,
|
||||||
installId = "install",
|
installId = "install",
|
||||||
appVersion = "1.0",
|
appVersion = "1.0",
|
||||||
platform = "Test",
|
platform = "Test",
|
||||||
exceptionType = "IllegalStateException",
|
exceptionType = "IllegalStateException",
|
||||||
exceptionMessage = "boom\nline",
|
exceptionMessage = "boom\nline\\nliteral\u001fseparator",
|
||||||
stackTrace = "stack\ntrace",
|
stackTrace = "stack\ntrace\u001erecord",
|
||||||
breadcrumbs = listOf(
|
breadcrumbs = listOf(
|
||||||
Breadcrumb("open", 1L, mapOf("screen" to "send")),
|
Breadcrumb("open|send", 1L, mapOf("screen:key" to "send,value|next")),
|
||||||
),
|
),
|
||||||
diagnosticsEnabledAtCapture = true,
|
diagnosticsEnabledAtCapture = true,
|
||||||
)
|
)
|
||||||
val decoded = CrashReportCodec.decode(CrashReportCodec.encode(original))
|
val decoded = CrashReportCodec.decode(CrashReportCodec.encode(original))
|
||||||
assertEquals(original, decoded)
|
assertEquals(original, decoded)
|
||||||
|
assertNull(CrashReportCodec.decode(CrashReportCodec.encode(original.copy(id = "../../escape"))))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashCodecMigratesLegacyV1Envelope() {
|
||||||
|
val raw = listOf(
|
||||||
|
"id=bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
|
||||||
|
"ts=42",
|
||||||
|
"install=legacy-install",
|
||||||
|
"app=0.9",
|
||||||
|
"platform=Desktop",
|
||||||
|
"type=IllegalStateException",
|
||||||
|
"message=first\\nsecond",
|
||||||
|
"stack=frame one\\nframe two",
|
||||||
|
"diag=1",
|
||||||
|
"schema=1",
|
||||||
|
"crumbs=1|opened|screen:send",
|
||||||
|
).joinToString("\u001f")
|
||||||
|
|
||||||
|
val report = requireNotNull(CrashReportCodec.decode(raw))
|
||||||
|
|
||||||
|
assertEquals("first\nsecond", report.exceptionMessage)
|
||||||
|
assertEquals("frame one\nframe two", report.stackTrace)
|
||||||
|
assertEquals(true, report.diagnosticsEnabledAtCapture)
|
||||||
|
assertEquals(
|
||||||
|
listOf(Breadcrumb("opened", 1, mapOf("screen" to "send"))),
|
||||||
|
report.breadcrumbs,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -150,13 +646,165 @@ class DiagnosticsTest {
|
|||||||
|
|
||||||
val optedIn = reporter.capture(RuntimeException("a"), diagnosticsEnabledOverride = true)
|
val optedIn = reporter.capture(RuntimeException("a"), diagnosticsEnabledOverride = true)
|
||||||
reporter.capture(RuntimeException("b"), diagnosticsEnabledOverride = false)
|
reporter.capture(RuntimeException("b"), diagnosticsEnabledOverride = false)
|
||||||
assertEquals(2, store.list().size)
|
assertEquals(1, store.list().size)
|
||||||
|
|
||||||
reporter.flushPending()
|
reporter.flushPending()
|
||||||
assertEquals(1, transport.crashes.size)
|
assertEquals(1, transport.crashes.size)
|
||||||
assertEquals(optedIn.id, transport.crashes.single().id)
|
assertEquals(optedIn.id, transport.crashes.single().id)
|
||||||
|
assertTrue(store.list().isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashReporterRetainsCrashWhenDeliveryIsUnavailable() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val store = InMemoryPendingCrashStore()
|
||||||
|
val reporter = CrashReporter(
|
||||||
|
store = store,
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = NoOpDiagnosticsTransport(),
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
reporter.capture(RuntimeException("boom"), diagnosticsEnabledOverride = true)
|
||||||
|
|
||||||
|
reporter.flushPending()
|
||||||
assertEquals(1, store.list().size)
|
assertEquals(1, store.list().size)
|
||||||
assertFalse(store.list().single().diagnosticsEnabledAtCapture)
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashReporterDropsPermanentPayloadFailuresAndStopsAfterTransientFailures() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val permanentTransport = RecordingDiagnosticsTransport().apply {
|
||||||
|
crashResult = Result.failure(DiagnosticsHttpException(400, "/v1/crashes"))
|
||||||
|
}
|
||||||
|
val permanentStore = InMemoryPendingCrashStore()
|
||||||
|
val permanentReporter = CrashReporter(
|
||||||
|
store = permanentStore,
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = permanentTransport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
permanentReporter.capture(RuntimeException("invalid"), diagnosticsEnabledOverride = true)
|
||||||
|
permanentReporter.flushPending()
|
||||||
|
assertTrue(permanentStore.list().isEmpty())
|
||||||
|
|
||||||
|
val transientTransport = RecordingDiagnosticsTransport().apply {
|
||||||
|
crashResult = Result.failure(IllegalStateException("offline"))
|
||||||
|
}
|
||||||
|
val transientStore = InMemoryPendingCrashStore()
|
||||||
|
val transientReporter = CrashReporter(
|
||||||
|
store = transientStore,
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transientTransport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
repeat(2) {
|
||||||
|
transientReporter.capture(RuntimeException("offline-$it"), diagnosticsEnabledOverride = true)
|
||||||
|
}
|
||||||
|
transientReporter.flushPending()
|
||||||
|
assertEquals(1, transientTransport.crashes.size)
|
||||||
|
assertEquals(2, transientStore.list().size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashReporterResolvesStartupConsentBeforeUploading() = runTest {
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val store = InMemoryPendingCrashStore()
|
||||||
|
val reporter = CrashReporter(
|
||||||
|
store = store,
|
||||||
|
preferencesRepository = fakePrefs(diagnosticsEnabled = true),
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
val startupCrash = reporter.capture(RuntimeException("startup"))
|
||||||
|
assertNull(startupCrash.diagnosticsEnabledAtCapture)
|
||||||
|
|
||||||
|
reporter.flushPending()
|
||||||
|
|
||||||
|
assertEquals(true, transport.crashes.single().diagnosticsEnabledAtCapture)
|
||||||
|
assertEquals("test-install", transport.crashes.single().installId)
|
||||||
|
assertTrue(store.list().isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashReporterDoesNotRetroactivelyUploadAnOptedOutStartupCrash() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = false)
|
||||||
|
val transport = RecordingDiagnosticsTransport()
|
||||||
|
val store = InMemoryPendingCrashStore()
|
||||||
|
val reporter = CrashReporter(
|
||||||
|
store = store,
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
reporter.capture(RuntimeException("startup"))
|
||||||
|
|
||||||
|
reporter.flushPending()
|
||||||
|
assertTrue(store.list().isEmpty())
|
||||||
|
preferences.setDiagnosticsEnabled(true)
|
||||||
|
reporter.flushPending()
|
||||||
|
|
||||||
|
assertTrue(transport.crashes.isEmpty())
|
||||||
|
assertTrue(store.list().isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun crashReporterStopsAFlushWhenTheUserOptsOut() = runTest {
|
||||||
|
val preferences = fakePrefs(diagnosticsEnabled = true)
|
||||||
|
val firstSendStarted = CompletableDeferred<Unit>()
|
||||||
|
val releaseFirstSend = CompletableDeferred<Unit>()
|
||||||
|
val sentIds = mutableListOf<String>()
|
||||||
|
val transport = object : DiagnosticsTransport {
|
||||||
|
override suspend fun sendEvents(batch: TelemetryBatch) = Result.success(Unit)
|
||||||
|
override suspend fun sendBugReport(report: BugReport) = Result.success(Unit)
|
||||||
|
override suspend fun sendCrash(report: CrashReport): Result<Unit> {
|
||||||
|
sentIds += report.id
|
||||||
|
if (sentIds.size == 1) {
|
||||||
|
firstSendStarted.complete(Unit)
|
||||||
|
releaseFirstSend.await()
|
||||||
|
}
|
||||||
|
return Result.success(Unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val store = InMemoryPendingCrashStore()
|
||||||
|
val reporter = CrashReporter(
|
||||||
|
store = store,
|
||||||
|
preferencesRepository = preferences,
|
||||||
|
transport = transport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
scope = TestScope(UnconfinedTestDispatcher(testScheduler)),
|
||||||
|
)
|
||||||
|
reporter.startObservingPreferences()
|
||||||
|
advanceUntilIdle()
|
||||||
|
repeat(2) {
|
||||||
|
reporter.capture(RuntimeException("crash-$it"), diagnosticsEnabledOverride = true)
|
||||||
|
}
|
||||||
|
|
||||||
|
val flush = launch { reporter.flushPending() }
|
||||||
|
firstSendStarted.await()
|
||||||
|
preferences.setDiagnosticsEnabled(false)
|
||||||
|
runCurrent()
|
||||||
|
releaseFirstSend.complete(Unit)
|
||||||
|
flush.join()
|
||||||
|
|
||||||
|
assertEquals(1, sentIds.size)
|
||||||
|
assertTrue(store.list().isEmpty())
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -174,6 +822,29 @@ class DiagnosticsTest {
|
|||||||
assertTrue(service.submit(BugReportDraft("what", ""), device()).isFailure)
|
assertTrue(service.submit(BugReportDraft("what", ""), device()).isFailure)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun bugReportConvertsTransportExceptionsToFailure() = runTest {
|
||||||
|
val throwingTransport = object : DiagnosticsTransport {
|
||||||
|
override suspend fun sendEvents(batch: TelemetryBatch) = Result.success(Unit)
|
||||||
|
override suspend fun sendCrash(report: CrashReport) = Result.success(Unit)
|
||||||
|
override suspend fun sendBugReport(report: BugReport): Result<Unit> {
|
||||||
|
throw IllegalStateException("offline")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val service = BugReportService(
|
||||||
|
preferencesRepository = fakePrefs(),
|
||||||
|
transport = throwingTransport,
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
)
|
||||||
|
|
||||||
|
val result = service.submit(BugReportDraft("what", "expected"), device())
|
||||||
|
|
||||||
|
assertTrue(result.isFailure)
|
||||||
|
assertEquals("offline", result.exceptionOrNull()?.message)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun bugReportSubmitsWithRedactedLogsRegardlessOfDiagnostics() = runTest {
|
fun bugReportSubmitsWithRedactedLogsRegardlessOfDiagnostics() = runTest {
|
||||||
val preferences = fakePrefs(diagnosticsEnabled = false)
|
val preferences = fakePrefs(diagnosticsEnabled = false)
|
||||||
@@ -205,6 +876,43 @@ class DiagnosticsTest {
|
|||||||
assertEquals("test-install", report.installId)
|
assertEquals("test-install", report.installId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun bugReportRedactsLogsBeforeApplyingUtf8Limit() {
|
||||||
|
val secret = "abcdefghijklmnopqrstuvwxyz012345"
|
||||||
|
val rawLogs = "ticket=$secret\n".repeat(6_000) + "tail-marker"
|
||||||
|
val service = BugReportService(
|
||||||
|
preferencesRepository = fakePrefs(),
|
||||||
|
transport = RecordingDiagnosticsTransport(),
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
logReader = { rawLogs },
|
||||||
|
)
|
||||||
|
|
||||||
|
val logs = service.assemble(BugReportDraft("what", "expected"), device(), "install").logs
|
||||||
|
|
||||||
|
assertFalse(logs.contains(secret))
|
||||||
|
assertTrue(logs.endsWith("tail-marker"))
|
||||||
|
assertTrue(logs.encodeToByteArray().size <= BugReportService.MaxLogBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun bugReportLogLimitCountsUtf8Bytes() {
|
||||||
|
val service = BugReportService(
|
||||||
|
preferencesRepository = fakePrefs(),
|
||||||
|
transport = RecordingDiagnosticsTransport(),
|
||||||
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Test",
|
||||||
|
logReader = { "🙂".repeat(60_000) },
|
||||||
|
)
|
||||||
|
|
||||||
|
val logs = service.assemble(BugReportDraft("what", "expected"), device(), "install").logs
|
||||||
|
|
||||||
|
assertEquals(BugReportService.MaxLogBytes, logs.encodeToByteArray().size)
|
||||||
|
assertEquals(BugReportService.MaxLogBytes, service.previewLogBytes())
|
||||||
|
}
|
||||||
|
|
||||||
private fun fakePrefs(diagnosticsEnabled: Boolean = false) = FakePreferencesRepository(
|
private fun fakePrefs(diagnosticsEnabled: Boolean = false) = FakePreferencesRepository(
|
||||||
AppPreferences(
|
AppPreferences(
|
||||||
username = "User",
|
username = "User",
|
||||||
@@ -228,4 +936,15 @@ private class InMemoryPendingCrashStore : PendingCrashStore {
|
|||||||
override fun delete(id: String) {
|
override fun delete(id: String) {
|
||||||
items.remove(id)
|
items.remove(id)
|
||||||
}
|
}
|
||||||
|
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
|
||||||
|
items.values
|
||||||
|
.sortedByDescending { it.timestampMillis }
|
||||||
|
.drop(maxCount)
|
||||||
|
.map(CrashReport::id)
|
||||||
|
.forEach(items::remove)
|
||||||
|
items.values
|
||||||
|
.filter { it.timestampMillis < olderThanTimestampMillis }
|
||||||
|
.map(CrashReport::id)
|
||||||
|
.forEach(items::remove)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,13 +10,16 @@ import com.vnidrop.app.core.ShareAccessPolicy
|
|||||||
import com.vnidrop.app.core.Transfer
|
import com.vnidrop.app.core.Transfer
|
||||||
import com.vnidrop.app.core.TransferDirection
|
import com.vnidrop.app.core.TransferDirection
|
||||||
import com.vnidrop.app.core.TransferStatus
|
import com.vnidrop.app.core.TransferStatus
|
||||||
|
import com.vnidrop.app.diagnostics.BreadcrumbBuffer
|
||||||
|
import com.vnidrop.app.diagnostics.BugReportService
|
||||||
|
import com.vnidrop.app.diagnostics.DiagnosticsTransport
|
||||||
|
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport
|
||||||
|
import com.vnidrop.app.diagnostics.RecordingDiagnosticsTransport
|
||||||
import com.vnidrop.app.feature.app.AppViewModel
|
import com.vnidrop.app.feature.app.AppViewModel
|
||||||
import com.vnidrop.app.feature.receive.ReceiveHistoryDeleteTarget
|
import com.vnidrop.app.feature.receive.ReceiveHistoryDeleteTarget
|
||||||
import com.vnidrop.app.feature.receive.ReceiveViewModel
|
import com.vnidrop.app.feature.receive.ReceiveViewModel
|
||||||
import com.vnidrop.app.feature.send.SendViewModel
|
import com.vnidrop.app.feature.send.SendViewModel
|
||||||
import com.vnidrop.app.diagnostics.BugReportService
|
import com.vnidrop.app.feature.settings.SettingsSection
|
||||||
import com.vnidrop.app.diagnostics.NoOpDiagnosticsTransport
|
|
||||||
import com.vnidrop.app.diagnostics.BreadcrumbBuffer
|
|
||||||
import com.vnidrop.app.feature.settings.SettingsViewModel
|
import com.vnidrop.app.feature.settings.SettingsViewModel
|
||||||
import com.vnidrop.app.notifications.NotificationPermission
|
import com.vnidrop.app.notifications.NotificationPermission
|
||||||
import com.vnidrop.app.preferences.AppPreferences
|
import com.vnidrop.app.preferences.AppPreferences
|
||||||
@@ -165,6 +168,24 @@ class ViewModelsTest {
|
|||||||
assertEquals("", viewModel.state.value.bugExpected)
|
assertEquals("", viewModel.state.value.bugExpected)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun settingsKeepsBugReportWhenDeliveryIsUnavailable() = runTest {
|
||||||
|
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
||||||
|
val viewModel = settingsViewModel(transport = NoOpDiagnosticsTransport())
|
||||||
|
advanceUntilIdle()
|
||||||
|
viewModel.selectSection(SettingsSection.BugReport)
|
||||||
|
viewModel.setBugWhatHappened("Transfer stuck")
|
||||||
|
viewModel.setBugExpected("It should finish")
|
||||||
|
|
||||||
|
viewModel.submitBugReport()
|
||||||
|
advanceUntilIdle()
|
||||||
|
|
||||||
|
assertEquals("Transfer stuck", viewModel.state.value.bugWhatHappened)
|
||||||
|
assertEquals("It should finish", viewModel.state.value.bugExpected)
|
||||||
|
assertEquals(SettingsSection.BugReport, viewModel.state.value.selectedSection)
|
||||||
|
assertFalse(viewModel.state.value.isSubmittingBugReport)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun sendViewModelOwnsSelectedFileState() = runTest {
|
fun sendViewModelOwnsSelectedFileState() = runTest {
|
||||||
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
||||||
@@ -512,6 +533,7 @@ class ViewModelsTest {
|
|||||||
private fun settingsViewModel(
|
private fun settingsViewModel(
|
||||||
preferences: PreferencesRepository = preferences(),
|
preferences: PreferencesRepository = preferences(),
|
||||||
notifications: FakeNotificationService = FakeNotificationService(),
|
notifications: FakeNotificationService = FakeNotificationService(),
|
||||||
|
transport: DiagnosticsTransport = RecordingDiagnosticsTransport(),
|
||||||
) = SettingsViewModel(
|
) = SettingsViewModel(
|
||||||
environment(),
|
environment(),
|
||||||
{ DeviceInfo("Device", "Model", "OS", "Wi-Fi", "80%") },
|
{ DeviceInfo("Device", "Model", "OS", "Wi-Fi", "80%") },
|
||||||
@@ -521,7 +543,7 @@ class ViewModelsTest {
|
|||||||
UiMessageController(),
|
UiMessageController(),
|
||||||
BugReportService(
|
BugReportService(
|
||||||
preferencesRepository = preferences,
|
preferencesRepository = preferences,
|
||||||
transport = NoOpDiagnosticsTransport(),
|
transport = transport,
|
||||||
breadcrumbs = BreadcrumbBuffer(),
|
breadcrumbs = BreadcrumbBuffer(),
|
||||||
appVersion = "1.0",
|
appVersion = "1.0",
|
||||||
platform = "Test",
|
platform = "Test",
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
package com.vnidrop.app.diagnostics
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import kotlinx.cinterop.BetaInteropApi
|
||||||
import kotlinx.cinterop.ExperimentalForeignApi
|
import kotlinx.cinterop.ExperimentalForeignApi
|
||||||
import kotlinx.cinterop.addressOf
|
import kotlinx.cinterop.addressOf
|
||||||
import kotlinx.cinterop.convert
|
import kotlinx.cinterop.convert
|
||||||
import kotlinx.cinterop.usePinned
|
import kotlinx.cinterop.usePinned
|
||||||
import platform.Foundation.NSData
|
import platform.Foundation.NSData
|
||||||
import platform.Foundation.NSFileManager
|
import platform.Foundation.NSFileManager
|
||||||
import platform.Foundation.NSString
|
|
||||||
import platform.Foundation.NSUTF8StringEncoding
|
|
||||||
import platform.Foundation.create
|
import platform.Foundation.create
|
||||||
import platform.Foundation.dataUsingEncoding
|
|
||||||
import platform.Foundation.dataWithContentsOfFile
|
import platform.Foundation.dataWithContentsOfFile
|
||||||
import platform.Foundation.writeToFile
|
import platform.Foundation.writeToFile
|
||||||
import platform.posix.memcpy
|
import platform.posix.memcpy
|
||||||
@@ -25,10 +23,11 @@ private class IosPendingCrashStore(
|
|||||||
private val directory = appDataDir.trimEnd('/') + "/diagnostics/crashes"
|
private val directory = appDataDir.trimEnd('/') + "/diagnostics/crashes"
|
||||||
|
|
||||||
override fun write(report: CrashReport) {
|
override fun write(report: CrashReport) {
|
||||||
|
if (!isValidDiagnosticId(report.id)) return
|
||||||
ensureDirectory()
|
ensureDirectory()
|
||||||
val path = "$directory/${report.id}.crash"
|
val path = "$directory/${report.id}.crash"
|
||||||
val payload = CrashReportCodec.encode(report)
|
val payload = CrashReportCodec.encode(report)
|
||||||
val data = (payload as NSString).dataUsingEncoding(NSUTF8StringEncoding) ?: return
|
val data = payload.encodeToByteArray().toNSData()
|
||||||
data.writeToFile(path, atomically = true)
|
data.writeToFile(path, atomically = true)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,14 +45,47 @@ private class IosPendingCrashStore(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun delete(id: String) {
|
override fun delete(id: String) {
|
||||||
|
if (!isValidDiagnosticId(id)) return
|
||||||
fileManager.removeItemAtPath("$directory/$id.crash", null)
|
fileManager.removeItemAtPath("$directory/$id.crash", null)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
|
||||||
|
require(maxCount > 0) { "maxCount must be positive" }
|
||||||
|
ensureDirectory()
|
||||||
|
val reports = fileManager.contentsOfDirectoryAtPath(directory, null).orEmpty()
|
||||||
|
.filterIsInstance<String>()
|
||||||
|
.filter { it.endsWith(".crash") }
|
||||||
|
.mapNotNull { name ->
|
||||||
|
val path = "$directory/$name"
|
||||||
|
val report = NSData.dataWithContentsOfFile(path)
|
||||||
|
?.toUtf8String()
|
||||||
|
?.let(CrashReportCodec::decode)
|
||||||
|
if (report == null) {
|
||||||
|
fileManager.removeItemAtPath(path, null)
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
name to report
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.sortedByDescending { (_, report) -> report.timestampMillis }
|
||||||
|
reports.forEachIndexed { index, (name, report) ->
|
||||||
|
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) {
|
||||||
|
fileManager.removeItemAtPath("$directory/$name", null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun ensureDirectory() {
|
private fun ensureDirectory() {
|
||||||
fileManager.createDirectoryAtPath(directory, withIntermediateDirectories = true, attributes = null, error = null)
|
fileManager.createDirectoryAtPath(directory, withIntermediateDirectories = true, attributes = null, error = null)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
|
||||||
|
private fun ByteArray.toNSData(): NSData =
|
||||||
|
usePinned { pinned ->
|
||||||
|
NSData.create(bytes = pinned.addressOf(0), length = size.toULong())
|
||||||
|
}
|
||||||
|
|
||||||
@OptIn(ExperimentalForeignApi::class)
|
@OptIn(ExperimentalForeignApi::class)
|
||||||
private fun NSData.toUtf8String(): String {
|
private fun NSData.toUtf8String(): String {
|
||||||
val size = length.toInt()
|
val size = length.toInt()
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import kotlinx.cinterop.BetaInteropApi
|
||||||
|
import kotlinx.cinterop.ExperimentalForeignApi
|
||||||
|
import kotlinx.cinterop.addressOf
|
||||||
|
import kotlinx.cinterop.convert
|
||||||
|
import kotlinx.cinterop.usePinned
|
||||||
|
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||||
|
import platform.Foundation.NSData
|
||||||
|
import platform.Foundation.NSHTTPURLResponse
|
||||||
|
import platform.Foundation.NSMutableURLRequest
|
||||||
|
import platform.Foundation.NSURL
|
||||||
|
import platform.Foundation.NSURLSession
|
||||||
|
import platform.Foundation.create
|
||||||
|
import platform.Foundation.dataTaskWithRequest
|
||||||
|
import platform.Foundation.setHTTPBody
|
||||||
|
import platform.Foundation.setHTTPMethod
|
||||||
|
import platform.Foundation.setValue
|
||||||
|
import platform.posix.memcpy
|
||||||
|
import kotlin.coroutines.resume
|
||||||
|
|
||||||
|
@OptIn(ExperimentalForeignApi::class)
|
||||||
|
actual suspend fun platformHttpPost(
|
||||||
|
url: String,
|
||||||
|
headers: Map<String, String>,
|
||||||
|
bodyUtf8: String,
|
||||||
|
): PlatformHttpResponse = suspendCancellableCoroutine { cont ->
|
||||||
|
val nsUrl = NSURL.URLWithString(url)
|
||||||
|
if (nsUrl == null) {
|
||||||
|
cont.resume(PlatformHttpResponse(statusCode = 0, body = "invalid_url"))
|
||||||
|
return@suspendCancellableCoroutine
|
||||||
|
}
|
||||||
|
val request = NSMutableURLRequest.requestWithURL(nsUrl).apply {
|
||||||
|
setHTTPMethod("POST")
|
||||||
|
setValue("application/json; charset=utf-8", forHTTPHeaderField = "Content-Type")
|
||||||
|
headers.forEach { (key, value) ->
|
||||||
|
setValue(value, forHTTPHeaderField = key)
|
||||||
|
}
|
||||||
|
setHTTPBody(bodyUtf8.encodeToByteArray().toNSData())
|
||||||
|
}
|
||||||
|
val task = NSURLSession.sharedSession.dataTaskWithRequest(request) { data, response, error ->
|
||||||
|
if (!cont.isActive) return@dataTaskWithRequest
|
||||||
|
if (error != null) {
|
||||||
|
val message = error.localizedDescription
|
||||||
|
cont.resume(PlatformHttpResponse(statusCode = 0, body = message))
|
||||||
|
return@dataTaskWithRequest
|
||||||
|
}
|
||||||
|
val http = response as? NSHTTPURLResponse
|
||||||
|
val status = http?.statusCode?.toInt() ?: 0
|
||||||
|
val body = data?.toUtf8String().orEmpty()
|
||||||
|
cont.resume(PlatformHttpResponse(statusCode = status, body = body))
|
||||||
|
}
|
||||||
|
cont.invokeOnCancellation { task.cancel() }
|
||||||
|
task.resume()
|
||||||
|
}
|
||||||
|
|
||||||
|
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
|
||||||
|
private fun ByteArray.toNSData(): NSData =
|
||||||
|
usePinned { pinned ->
|
||||||
|
NSData.create(bytes = pinned.addressOf(0), length = size.toULong())
|
||||||
|
}
|
||||||
|
|
||||||
|
@OptIn(ExperimentalForeignApi::class)
|
||||||
|
private fun NSData.toUtf8String(): String {
|
||||||
|
val size = length.toInt()
|
||||||
|
if (size == 0) return ""
|
||||||
|
val result = ByteArray(size)
|
||||||
|
val source = bytes ?: return ""
|
||||||
|
result.usePinned { pinned ->
|
||||||
|
memcpy(pinned.addressOf(0), source, size.convert())
|
||||||
|
}
|
||||||
|
return result.decodeToString()
|
||||||
|
}
|
||||||
@@ -13,8 +13,16 @@ private class JvmPendingCrashStore(
|
|||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
override fun write(report: CrashReport) {
|
override fun write(report: CrashReport) {
|
||||||
|
if (!isValidDiagnosticId(report.id)) return
|
||||||
directory.mkdirs()
|
directory.mkdirs()
|
||||||
File(directory, "${report.id}.crash").writeText(CrashReportCodec.encode(report), StandardCharsets.UTF_8)
|
val target = File(directory, "${report.id}.crash")
|
||||||
|
val temporary = File(directory, ".${report.id}.tmp")
|
||||||
|
val payload = CrashReportCodec.encode(report)
|
||||||
|
temporary.writeText(payload, StandardCharsets.UTF_8)
|
||||||
|
if (!temporary.renameTo(target)) {
|
||||||
|
target.writeText(payload, StandardCharsets.UTF_8)
|
||||||
|
temporary.delete()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
@@ -31,6 +39,34 @@ private class JvmPendingCrashStore(
|
|||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
override fun delete(id: String) {
|
override fun delete(id: String) {
|
||||||
|
if (!isValidDiagnosticId(id)) return
|
||||||
File(directory, "$id.crash").delete()
|
File(directory, "$id.crash").delete()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
override fun prune(olderThanTimestampMillis: Long, maxCount: Int) {
|
||||||
|
require(maxCount > 0) { "maxCount must be positive" }
|
||||||
|
if (!directory.isDirectory) return
|
||||||
|
directory.listFiles { file -> file.isFile && file.name.endsWith(".tmp") }
|
||||||
|
.orEmpty()
|
||||||
|
.forEach(File::delete)
|
||||||
|
val reports = directory
|
||||||
|
.listFiles { file -> file.isFile && file.name.endsWith(".crash") }
|
||||||
|
.orEmpty()
|
||||||
|
.mapNotNull { file ->
|
||||||
|
val report = runCatching {
|
||||||
|
CrashReportCodec.decode(file.readText(StandardCharsets.UTF_8))
|
||||||
|
}.getOrNull()
|
||||||
|
if (report == null) {
|
||||||
|
file.delete()
|
||||||
|
null
|
||||||
|
} else {
|
||||||
|
file to report
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.sortedByDescending { (_, report) -> report.timestampMillis }
|
||||||
|
reports.forEachIndexed { index, (file, report) ->
|
||||||
|
if (index >= maxCount || report.timestampMillis < olderThanTimestampMillis) file.delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import java.io.BufferedReader
|
||||||
|
import java.io.InputStreamReader
|
||||||
|
import java.net.HttpURLConnection
|
||||||
|
import java.net.URI
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
|
||||||
|
actual suspend fun platformHttpPost(
|
||||||
|
url: String,
|
||||||
|
headers: Map<String, String>,
|
||||||
|
bodyUtf8: String,
|
||||||
|
): PlatformHttpResponse = withContext(Dispatchers.IO) {
|
||||||
|
val connection = (URI(url).toURL().openConnection() as HttpURLConnection).apply {
|
||||||
|
requestMethod = "POST"
|
||||||
|
doOutput = true
|
||||||
|
connectTimeout = 15_000
|
||||||
|
readTimeout = 30_000
|
||||||
|
setRequestProperty("Content-Type", "application/json; charset=utf-8")
|
||||||
|
headers.forEach { (key, value) -> setRequestProperty(key, value) }
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
connection.outputStream.use { output ->
|
||||||
|
output.write(bodyUtf8.toByteArray(StandardCharsets.UTF_8))
|
||||||
|
}
|
||||||
|
val code = connection.responseCode
|
||||||
|
val stream = if (code in 200..299) connection.inputStream else connection.errorStream
|
||||||
|
val body = stream?.use { input ->
|
||||||
|
BufferedReader(InputStreamReader(input, StandardCharsets.UTF_8)).readText()
|
||||||
|
}.orEmpty()
|
||||||
|
PlatformHttpResponse(code, body)
|
||||||
|
} finally {
|
||||||
|
connection.disconnect()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package com.vnidrop.app.diagnostics
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.nio.file.Files
|
||||||
|
import kotlin.test.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertFalse
|
||||||
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
|
class PendingCrashStoreJvmTest {
|
||||||
|
@Test
|
||||||
|
fun replacesReportsAndPrunesOldCorruptAndTemporaryFiles() {
|
||||||
|
val root = Files.createTempDirectory("vnidrop-crash-store").toFile()
|
||||||
|
try {
|
||||||
|
val store = createPendingCrashStore(root.absolutePath)
|
||||||
|
val older = report("10000000-0000-4000-8000-000000000001", 1, "older")
|
||||||
|
val current = report("10000000-0000-4000-8000-000000000002", 2, "current")
|
||||||
|
store.write(older)
|
||||||
|
store.write(current)
|
||||||
|
store.write(current.copy(exceptionMessage = "replaced"))
|
||||||
|
|
||||||
|
val directory = File(root, "diagnostics/crashes")
|
||||||
|
File(directory, "corrupt.crash").writeText("not a crash envelope")
|
||||||
|
File(directory, ".orphan.tmp").writeText("partial")
|
||||||
|
store.write(current.copy(id = "../../escape"))
|
||||||
|
val escapedPath = File(directory, "../../escape.crash").canonicalFile
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
listOf("replaced", "older"),
|
||||||
|
store.list().map(CrashReport::exceptionMessage),
|
||||||
|
)
|
||||||
|
store.prune(olderThanTimestampMillis = 0, maxCount = 1)
|
||||||
|
|
||||||
|
assertEquals(listOf("replaced"), store.list().map(CrashReport::exceptionMessage))
|
||||||
|
assertFalse(File(directory, "corrupt.crash").exists())
|
||||||
|
assertFalse(File(directory, ".orphan.tmp").exists())
|
||||||
|
assertFalse(escapedPath.exists())
|
||||||
|
assertTrue(directory.listFiles().orEmpty().all { it.parentFile == directory })
|
||||||
|
} finally {
|
||||||
|
root.deleteRecursively()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun report(id: String, timestampMillis: Long, message: String) = CrashReport(
|
||||||
|
id = id,
|
||||||
|
timestampMillis = timestampMillis,
|
||||||
|
installId = "install",
|
||||||
|
appVersion = "1.0",
|
||||||
|
platform = "Desktop",
|
||||||
|
exceptionType = "TestError",
|
||||||
|
exceptionMessage = message,
|
||||||
|
stackTrace = "stack",
|
||||||
|
breadcrumbs = emptyList(),
|
||||||
|
diagnosticsEnabledAtCapture = true,
|
||||||
|
)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user