ico/pingql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: improve sql queries

Browse Source
This commit is contained in:
nate
2026-04-09 04:48:50 +04:00
parent 89f0856a04
commit 91ca996e74
10 changed files with 495 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/status/src/data.ts
+175 -135
View File
@@ -53,63 +53,11 @@ export interface MonitorRow {
latency_history: Array<{ region: string; latency_ms: number | null; ts: string }>;
}
// Single SQL pass that produces all four uptime windows for a set of monitors.
// Reads only the rollup table; falls back to a pings aggregate when the rollup
// has nothing for these monitors yet (same pattern as loadMonitors).
export async function loadMultiWindowUptime(monitorIds: string[]): Promise<Record<string, MultiWindowUptime>> {
const empty: Record<string, MultiWindowUptime> = {};
if (monitorIds.length === 0) return empty;
for (const id of monitorIds) empty[id] = { d24: null, d7: null, d30: null, d90: null };
const ids = sql.array(monitorIds);
let rows = await sql<any[]>`
SELECT monitor_id,
(sum(up_count) FILTER (WHERE bucket_type='hourly' AND bucket_start > now() - interval '24 hours'))::float
/ NULLIF(sum(total) FILTER (WHERE bucket_type='hourly' AND bucket_start > now() - interval '24 hours'), 0) AS pct_24h,
(sum(up_count) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '7 days'))::float
/ NULLIF(sum(total) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '7 days'), 0) AS pct_7d,
(sum(up_count) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '30 days'))::float
/ NULLIF(sum(total) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '30 days'), 0) AS pct_30d,
(sum(up_count) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '90 days'))::float
/ NULLIF(sum(total) FILTER (WHERE bucket_type='daily' AND bucket_start > now() - interval '90 days'), 0) AS pct_90d
FROM monitor_uptime_rollup
WHERE monitor_id = ANY(${ids}::text[])
GROUP BY 1
`;
// Fallback when the rollup is empty: aggregate directly from pings. Bounded
// by the 90d window so it's still cheap.
if (rows.length === 0) {
rows = await sql<any[]>`
SELECT monitor_id,
(count(*) FILTER (WHERE up AND checked_at > now() - interval '24 hours'))::float
/ NULLIF(count(*) FILTER (WHERE checked_at > now() - interval '24 hours'), 0) AS pct_24h,
(count(*) FILTER (WHERE up AND checked_at > now() - interval '7 days'))::float
/ NULLIF(count(*) FILTER (WHERE checked_at > now() - interval '7 days'), 0) AS pct_7d,
(count(*) FILTER (WHERE up AND checked_at > now() - interval '30 days'))::float
/ NULLIF(count(*) FILTER (WHERE checked_at > now() - interval '30 days'), 0) AS pct_30d,
(count(*) FILTER (WHERE up AND checked_at > now() - interval '90 days'))::float
/ NULLIF(count(*) FILTER (WHERE checked_at > now() - interval '90 days'), 0) AS pct_90d
FROM pings
WHERE monitor_id = ANY(${ids}::text[])
AND checked_at > now() - interval '90 days'
GROUP BY 1
`;
}
const out = empty;
const toPct = (v: any): number | null => v == null ? null : +(Number(v) * 100).toFixed(2);
for (const r of rows) {
out[r.monitor_id] = {
d24: toPct(r.pct_24h),
d7: toPct(r.pct_7d),
d30: toPct(r.pct_30d),
d90: toPct(r.pct_90d),
};
}
return out;
}
// Multi-window uptime (24h / 7d / 30d / 90d) is now derived from the same
// rollup row set that loadMonitors pulls for the bar chart — see the in-JS
// aggregation pass below. This used to be a second SQL round-trip running
// four FILTER aggregates that redid arithmetic the raw bucket rows already
// contained.
export interface GroupRow {
id: string;
@@ -189,85 +137,186 @@ export async function loadMonitors(
});
}
// Step 3: uptime rollup buckets covering the requested window. The bar
// frequency + count are admin-controlled per page (independent of the
// multi-window uptime cells). We keep region in the result so JS can pick
// the fastest region per monitor and emit per-bucket latency from just
// that region (status pages are customer-facing — show the best line).
// Step 3: ONE unified rollup query covering everything we need:
// - bar chart: bucket_type = barFrequency, last barCount buckets
// - multi-window uptime: hourly back 24h + daily back 90d
// - latency sparkline: hourly back 30h
//
// Union of all of those is "hourly back N hours OR daily back N days" with
// N chosen to cover whichever consumer needs the widest window. The rows
// are then partitioned by purpose entirely in JS — no second round-trip,
// no duplicate FILTER aggregates inside Postgres.
const bucket: BucketType = barFrequency;
const count = Math.max(1, Math.min(180, barCount));
const truncUnit = bucket === "hourly" ? "hour" : "day";
const intervalLiteral = `${count} ${truncUnit}s`;
// Hourly span has to cover the latency sparkline (30h) AND the bar chart if
// it's hourly (up to 180h). +2h slack so the truncated bucket boundary at
// the start of the window is included even if we cross an hour during the
// request.
const hourlyBackHours = Math.max(30, bucket === "hourly" ? count : 0) + 2;
// Daily span has to cover multi-window uptime (90d) AND the bar chart if
// it's daily (up to 180d). +1d slack for the same reason.
const dailyBackDays = Math.max(90, bucket === "daily" ? count : 0) + 1;
const hourlyInterval = `${hourlyBackHours} hours`;
const dailyInterval = `${dailyBackDays} days`;
let rollupRows = await sql<any[]>`
SELECT monitor_id, region, bucket_start, total, up_count, avg_latency
SELECT monitor_id, region, bucket_type, bucket_start, total, up_count, avg_latency
FROM monitor_uptime_rollup
WHERE monitor_id = ANY(${sql.array(ids)}::text[])
AND bucket_type = ${bucket}
AND bucket_start > date_trunc(${truncUnit}, now()) - ${intervalLiteral}::interval
ORDER BY monitor_id, region, bucket_start ASC
AND (
(bucket_type = 'hourly' AND bucket_start > now() - ${hourlyInterval}::interval)
OR
(bucket_type = 'daily' AND bucket_start > now() - ${dailyInterval}::interval)
)
ORDER BY monitor_id, bucket_type, region, bucket_start ASC
`;
// Fallback: if the rollup table has nothing for any of these monitors in
// this window (e.g. the api hasn't backfilled yet, or the rollup job is
// silently broken), aggregate directly from pings. Bounded by the window so
// it stays cheap. Once the rollup catches up this branch never fires.
// either bucket type (cold deploy, broken job), aggregate directly from
// pings. Produces both bucket types via UNION ALL so downstream JS doesn't
// need to know which path it came from. Bounded by the wider of the two
// windows so it stays cheap. Once the rollup catches up this never fires.
if (rollupRows.length === 0) {
rollupRows = await sql<any[]>`
SELECT
monitor_id,
COALESCE(region, 'default') AS region,
date_trunc(${truncUnit}, checked_at) AS bucket_start,
count(*)::int AS total,
count(*) FILTER (WHERE up)::int AS up_count,
avg(latency_ms)::real AS avg_latency
FROM pings
WHERE monitor_id = ANY(${sql.array(ids)}::text[])
AND checked_at > date_trunc(${truncUnit}, now()) - ${intervalLiteral}::interval
GROUP BY 1, 2, 3
ORDER BY 1, 2, 3 ASC
(
SELECT
monitor_id,
COALESCE(region, 'default') AS region,
'hourly'::text AS bucket_type,
date_trunc('hour', checked_at) AS bucket_start,
count(*)::int AS total,
count(*) FILTER (WHERE up)::int AS up_count,
avg(latency_ms)::real AS avg_latency
FROM pings
WHERE monitor_id = ANY(${sql.array(ids)}::text[])
AND checked_at > now() - ${hourlyInterval}::interval
GROUP BY 1, 2, 4
)
UNION ALL
(
SELECT
monitor_id,
COALESCE(region, 'default') AS region,
'daily'::text AS bucket_type,
date_trunc('day', checked_at) AS bucket_start,
count(*)::int AS total,
count(*) FILTER (WHERE up)::int AS up_count,
avg(latency_ms)::real AS avg_latency
FROM pings
WHERE monitor_id = ANY(${sql.array(ids)}::text[])
AND checked_at > now() - ${dailyInterval}::interval
GROUP BY 1, 2, 4
)
`;
}
// Single pass over rollup rows builds three indices:
// indexed[mid][isoStart] → cross-region {total, up} for bar coloring
// regionLat[mid][region] → cross-window weighted latency for picking fastest region
// regionBucketLat[mid][region][isoStart] → per-bucket latency for the fastest-region tooltip lookup
const indexed: Record<string, Record<string, { total: number; up: number }>> = {};
const regionLat: Record<string, Record<string, { sum: number; n: number }>> = {};
const regionBucketLat: Record<string, Record<string, Record<string, number>>> = {};
// Single pass over the unified rows builds every index we need:
// barIndexed[mid][isoStart] → cross-region {total, up} for bar coloring (only rows of barFrequency)
// barRegionLat[mid][region] → weighted latency over the bar window for picking fastest region
// barRegionBucketLat[mid][region][iso] → per-bucket latency in the fastest region (only rows of barFrequency)
// windowTotals[mid][windowKey] → {up, total} per uptime window (24h/7d/30d/90d)
// latByMonitor[mid][] → 30h hourly latency sparkline rows
const barIndexed: Record<string, Record<string, { total: number; up: number }>> = {};
const barRegionLat: Record<string, Record<string, { sum: number; n: number }>> = {};
const barRegionBucketLat: Record<string, Record<string, Record<string, number>>> = {};
type WindowKey = "d24" | "d7" | "d30" | "d90";
const windowTotals: Record<string, Record<WindowKey, { up: number; total: number }>> = {};
const initWindowTotals = (mid: string) => {
if (!windowTotals[mid]) {
windowTotals[mid] = {
d24: { up: 0, total: 0 },
d7: { up: 0, total: 0 },
d30: { up: 0, total: 0 },
d90: { up: 0, total: 0 },
};
}
return windowTotals[mid]!;
};
const latByMonitor: Record<string, MonitorRow["latency_history"]> = {};
const nowMs = Date.now();
const ms24h = 24 * 3600_000;
const ms7d = 7 * 86_400_000;
const ms30d = 30 * 86_400_000;
const ms90d = 90 * 86_400_000;
const ms30h = 30 * 3600_000;
for (const r of rollupRows) {
const startIso = r.bucket_start instanceof Date ? r.bucket_start.toISOString() : String(r.bucket_start);
const startDate = r.bucket_start instanceof Date ? r.bucket_start : new Date(r.bucket_start);
const startIso = startDate.toISOString();
const startMs = startDate.getTime();
const total = Number(r.total);
const up = Number(r.up_count);
const avgLat = r.avg_latency == null ? null : Number(r.avg_latency);
const mid = r.monitor_id;
const bt: BucketType = r.bucket_type;
// Cross-region bucket totals (for bar coloring)
if (!indexed[r.monitor_id]) indexed[r.monitor_id] = {};
const slot = indexed[r.monitor_id]![startIso] ?? { total: 0, up: 0 };
slot.total += Number(r.total);
slot.up += Number(r.up_count);
indexed[r.monitor_id]![startIso] = slot;
// Bar chart accumulators — only rows matching the configured bar frequency.
if (bt === bucket) {
if (!barIndexed[mid]) barIndexed[mid] = {};
const slot = barIndexed[mid]![startIso] ?? { total: 0, up: 0 };
slot.total += total;
slot.up += up;
barIndexed[mid]![startIso] = slot;
// Per-region latency tracking
if (r.avg_latency != null && Number(r.total) > 0) {
if (!regionLat[r.monitor_id]) regionLat[r.monitor_id] = {};
const acc = regionLat[r.monitor_id]![r.region] ?? { sum: 0, n: 0 };
acc.sum += Number(r.avg_latency) * Number(r.total);
acc.n += Number(r.total);
regionLat[r.monitor_id]![r.region] = acc;
if (avgLat != null && total > 0) {
if (!barRegionLat[mid]) barRegionLat[mid] = {};
const acc = barRegionLat[mid]![r.region] ?? { sum: 0, n: 0 };
acc.sum += avgLat * total;
acc.n += total;
barRegionLat[mid]![r.region] = acc;
if (!regionBucketLat[r.monitor_id]) regionBucketLat[r.monitor_id] = {};
if (!regionBucketLat[r.monitor_id]![r.region]) regionBucketLat[r.monitor_id]![r.region] = {};
regionBucketLat[r.monitor_id]![r.region]![startIso] = Math.round(Number(r.avg_latency));
if (!barRegionBucketLat[mid]) barRegionBucketLat[mid] = {};
if (!barRegionBucketLat[mid]![r.region]) barRegionBucketLat[mid]![r.region] = {};
barRegionBucketLat[mid]![r.region]![startIso] = Math.round(avgLat);
}
}
// Multi-window uptime accumulators. 24h uses hourly buckets; 7d/30d/90d
// use daily buckets — same as the old loadMultiWindowUptime SQL did.
// Strict `<` to match the old SQL's `bucket_start > now() - interval`.
const wt = initWindowTotals(mid);
if (bt === "hourly" && nowMs - startMs < ms24h) {
wt.d24.up += up; wt.d24.total += total;
}
if (bt === "daily") {
const age = nowMs - startMs;
if (age < ms7d) { wt.d7.up += up; wt.d7.total += total; }
if (age < ms30d) { wt.d30.up += up; wt.d30.total += total; }
if (age < ms90d) { wt.d90.up += up; wt.d90.total += total; }
}
// 30h hourly latency sparkline.
if (bt === "hourly" && nowMs - startMs < ms30h) {
if (!latByMonitor[mid]) latByMonitor[mid] = [];
latByMonitor[mid]!.push({
region: r.region,
latency_ms: avgLat == null ? null : Math.round(avgLat),
ts: startIso,
});
}
}
// Pick the fastest region per monitor (lowest weighted average latency over
// the whole window). All per-bucket latency display falls back to this
// region's per-bucket numbers; the per-monitor avg_latency uses the same.
// Sort the latency sparkline rows by ts ASC per monitor (the unified query
// sorts by bucket_type then region then bucket_start, so the per-monitor
// hourly subset is already ordered within a region but interleaved across
// regions — this normalises it the same way the old separate query did).
for (const mid of Object.keys(latByMonitor)) {
latByMonitor[mid]!.sort((a, b) => a.ts.localeCompare(b.ts));
}
// Pick the fastest region per monitor over the bar window (lowest weighted
// average latency). Per-bucket latency display + the per-monitor avg_latency
// both come from the chosen region.
const fastestRegionByMonitor: Record<string, string | null> = {};
const fastestLatency: Record<string, number | null> = {};
for (const id of ids) {
let bestRegion: string | null = null;
let bestAvg = Infinity;
const regions = regionLat[id] ?? {};
const regions = barRegionLat[id] ?? {};
for (const [region, acc] of Object.entries(regions)) {
if (acc.n === 0) continue;
const avg = acc.sum / acc.n;
@@ -278,8 +327,8 @@ export async function loadMonitors(
}
// Generate the full sequence of expected bucket timestamps so empty bars
// render as "no data" instead of disappearing entirely. Truncate `now()` to
// the unit so the slot boundaries line up with what the rollup writes.
// render as "no data" instead of disappearing entirely. Truncate `now()`
// to the unit so the slot boundaries line up with what the rollup writes.
const bucketMs = bucket === "hourly" ? 3600_000 : 86_400_000;
const truncate = (d: Date): Date => {
const t = new Date(d);
@@ -294,9 +343,9 @@ export async function loadMonitors(
}
const bucketsByMonitor: Record<string, MonitorRow["buckets"]> = {};
for (const id of ids) {
const slotMap = indexed[id] ?? {};
const slotMap = barIndexed[id] ?? {};
const bestRegion = fastestRegionByMonitor[id];
const fastestBuckets = bestRegion ? regionBucketLat[id]?.[bestRegion] ?? {} : {};
const fastestBuckets = bestRegion ? barRegionBucketLat[id]?.[bestRegion] ?? {} : {};
bucketsByMonitor[id] = slotIsos.map((iso) => {
const hit = slotMap[iso];
const lat = fastestBuckets[iso] ?? null;
@@ -306,28 +355,19 @@ export async function loadMonitors(
});
}
// Step 4: multi-window uptime row (24h / 7d / 30d / 90d) per monitor.
const multiWindow = await loadMultiWindowUptime(ids);
// Step 5: tiny recent latency history for the sparkline (last 30 hourly buckets).
const latRows = await sql<any[]>`
SELECT monitor_id, region, bucket_start, avg_latency
FROM monitor_uptime_rollup
WHERE monitor_id = ANY(${sql.array(ids)}::text[])
AND bucket_type = 'hourly'
AND bucket_start > now() - interval '30 hours'
ORDER BY monitor_id, bucket_start ASC
`;
const latencyByMonitorList: Record<string, MonitorRow["latency_history"]> = {};
for (const r of latRows) {
if (!latencyByMonitorList[r.monitor_id]) latencyByMonitorList[r.monitor_id] = [];
latencyByMonitorList[r.monitor_id]!.push({
region: r.region,
latency_ms: r.avg_latency != null ? Math.round(r.avg_latency) : null,
ts: r.bucket_start instanceof Date ? r.bucket_start.toISOString() : String(r.bucket_start),
});
// Multi-window uptime is a straight read from the windowTotals accumulator.
const multiWindow: Record<string, MultiWindowUptime> = {};
const toPct = (up: number, total: number): number | null =>
total > 0 ? +(100 * up / total).toFixed(2) : null;
for (const id of ids) {
const wt = windowTotals[id];
multiWindow[id] = wt
? { d24: toPct(wt.d24.up, wt.d24.total), d7: toPct(wt.d7.up, wt.d7.total), d30: toPct(wt.d30.up, wt.d30.total), d90: toPct(wt.d90.up, wt.d90.total) }
: { d24: null, d7: null, d30: null, d90: null };
}
const latencyByMonitorList = latByMonitor;
return monitorRows.map((m) => {
const region_states = stateByMonitor[m.id] ?? [];
let current_state: MonitorRow["current_state"] = "unknown";