From 5730a3cb838f1decd2265210cba7a51784fd8cac Mon Sep 17 00:00:00 2001 From: M1 Date: Wed, 18 Mar 2026 12:52:17 +0400 Subject: [PATCH] fix: replace reqwest with curl subprocess for reliable hard timeouts --- apps/monitor/src/runner.rs | 152 ++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 63 deletions(-) diff --git a/apps/monitor/src/runner.rs b/apps/monitor/src/runner.rs index 4349279..9a1d051 100644 --- a/apps/monitor/src/runner.rs +++ b/apps/monitor/src/runner.rs @@ -72,69 +72,20 @@ async fn run_check(client: &reqwest::Client, monitor: &Monitor, scheduled_at: Op let is_https = monitor.url.starts_with("https://"); let url_for_cert = monitor.url.clone(); - // Use blocking reqwest in a thread so OS-level socket timeouts actually work. - // Async reqwest with rustls/native-tls does not reliably cancel on TLS hangs. - let url = monitor.url.clone(); - let method_str = method.clone(); - let req_headers = monitor.request_headers.clone(); - let req_body = monitor.request_body.clone(); - let query_clone = monitor.query.clone(); - - let blocking_result = tokio::task::spawn_blocking(move || { - let block_client = reqwest::blocking::Client::builder() - .user_agent("PingQL-Monitor/0.1") - .connect_timeout(timeout) - .timeout(timeout) - .build()?; - - let req_method = reqwest::Method::from_bytes(method_str.as_bytes()) - .unwrap_or(reqwest::Method::GET); - - let mut req = block_client.request(req_method, &url); - - if let Some(ref headers) = req_headers { - for (k, v) in headers { - if let (Ok(name), Ok(value)) = ( - reqwest::header::HeaderName::from_bytes(k.as_bytes()), - reqwest::header::HeaderValue::from_str(v), - ) { - req = req.header(name, value); - } - } - } - - if let Some(ref body) = req_body { - req = req.body(body.clone()); - } - - let resp = req.send()?; - let status = resp.status(); - let headers: HashMap = resp.headers().iter() - .filter_map(|(k, v)| Some((k.to_string(), v.to_str().ok()?.to_string()))) - .collect(); - - const MAX_BODY_BYTES: usize = 10 * 1024 * 1024; - let body = { - let content_len = resp.content_length().unwrap_or(0) as usize; - if content_len > MAX_BODY_BYTES { - format!("[body truncated: Content-Length {} exceeds 10MB limit]", content_len) - } else { - let bytes = resp.bytes()?; - let truncated = &bytes[..bytes.len().min(MAX_BODY_BYTES)]; - String::from_utf8_lossy(truncated).into_owned() - } - }; - - Ok::<_, reqwest::Error>((status, headers, body, query_clone)) - }).await; + // Use curl subprocess — the only reliable way to enforce a hard timeout + // including TLS handshake on hosts that accept TCP but stall at TLS. + let timeout_secs = (timeout.as_millis() as f64 / 1000.0).max(1.0); + let curl_result = run_curl( + &monitor.url, + &method, + monitor.request_headers.as_ref(), + monitor.request_body.as_deref(), + timeout_secs, + ).await; let latency_ms = start.elapsed().as_millis() as u64; - let result = match blocking_result { - Err(e) => Err(e.to_string()), // spawn_blocking panic - Ok(Err(e)) => Err(e.to_string()), // reqwest error - Ok(Ok(v)) => Ok(v), - }; + let result = curl_result; match result { Err(e) => PingResult { @@ -148,8 +99,8 @@ async fn run_check(client: &reqwest::Client, monitor: &Monitor, scheduled_at: Op cert_expiry_days: None, meta: None, }, - Ok((status_raw, headers, body, query)) => { - let status = status_raw.as_u16(); + Ok((status_code, headers, body)) => { + let status = status_code; // Only attempt cert check after a successful response let cert_expiry_days = if is_https { @@ -164,8 +115,10 @@ async fn run_check(client: &reqwest::Client, monitor: &Monitor, scheduled_at: Op None }; + let query = &monitor.query; + // Evaluate query if present - let (up, query_error) = if let Some(q) = &query { + let (up, query_error) = if let Some(q) = query { let response = Response { status, body: body.clone(), @@ -208,6 +161,79 @@ async fn run_check(client: &reqwest::Client, monitor: &Monitor, scheduled_at: Op } } +/// Run an HTTP check via curl subprocess — the only reliable way to enforce +/// a hard timeout including TLS handshake on hosts that stall at that phase. +async fn run_curl( + url: &str, + method: &str, + headers: Option<&HashMap>, + body: Option<&str>, + timeout_secs: f64, +) -> Result<(u16, HashMap, String), String> { + let mut cmd = tokio::process::Command::new("curl"); + cmd.arg("--silent") + .arg("--show-error") + .arg("--include") // include response headers in output + .arg("--max-time").arg(format!("{:.1}", timeout_secs)) + .arg("--connect-timeout").arg(format!("{:.1}", timeout_secs)) + .arg("-X").arg(method) + .arg("--user-agent").arg("PingQL-Monitor/0.1") + .arg("--location"); // follow redirects + + if let Some(hdrs) = headers { + for (k, v) in hdrs { + cmd.arg("-H").arg(format!("{k}: {v}")); + } + } + + if let Some(b) = body { + cmd.arg("--data-raw").arg(b); + } + + cmd.arg(url); + + let output = cmd.output().await.map_err(|e| format!("curl exec error: {e}"))?; + + if !output.status.success() && output.stdout.is_empty() { + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + // curl exit 28 = timeout + let msg = if output.status.code() == Some(28) { + format!("timed out after {:.0}s", timeout_secs) + } else { + stderr.trim().to_string() + }; + return Err(msg); + } + + // Parse curl output: headers then blank line then body + let raw = String::from_utf8_lossy(&output.stdout).to_string(); + let mut parts = raw.splitn(2, "\r\n\r\n"); + let header_block = parts.next().unwrap_or(""); + let body_str = parts.next().unwrap_or("").to_string(); + + let mut lines = header_block.lines(); + let status_line = lines.next().unwrap_or("HTTP/1.1 0"); + let status_code: u16 = status_line.split_whitespace().nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let mut resp_headers = HashMap::new(); + for line in lines { + if let Some((k, v)) = line.split_once(':') { + resp_headers.insert(k.trim().to_lowercase(), v.trim().to_string()); + } + } + + const MAX_BODY: usize = 10 * 1024 * 1024; + let body_out = if body_str.len() > MAX_BODY { + format!("[body truncated: {} bytes]", body_str.len()) + } else { + body_str + }; + + Ok((status_code, resp_headers, body_out)) +} + /// Check SSL certificate expiry for a given HTTPS URL. /// Returns the number of days until the certificate expires. async fn check_cert_expiry(url: &str) -> Result> {