From f34c1fe2d3461001ae9a2bc56923910f6fa8f9ff Mon Sep 17 00:00:00 2001 From: lda Date: Sat, 10 Oct 2026 15:56:20 +0700 Subject: [PATCH] Configure console logs, OTLP logs, and traces independently --- docs/roadmap.md | 77 +++++++++++++++ docs/telemetry.md | 16 +++- wakey-agent/src/config.rs | 3 + wakey-agent/src/enroll.rs | 1 + wakey-control-plane/src/config/init.rs | 18 ++++ wakey-control-plane/src/config/resolve.rs | 3 + wakey-control-plane/src/config/types.rs | 18 ++++ wakey-control-plane/src/tracing.rs | 3 + wakey-core/src/telemetry.rs | 110 +++++++++++++++++++--- wakey-core/tests/telemetry.rs | 32 ++++++- 10 files changed, 263 insertions(+), 18 deletions(-) create mode 100644 docs/roadmap.md diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..833cea7 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,77 @@ +# Wakey Roadmap + +Last updated: 2026-10-10. This records agreed work and follow-ups, not a release +promise. Checked items are implemented; live validation is called out separately. + +## Completed Observability + +- [x] Shared optional OTLP setup for traces, logs, and metrics in wakey-core. +- [x] Basic/Bearer authorization supplied through a named environment variable. +- [x] OpenWrt UCI configuration and systemd environment-file support, with setup guides. +- [x] Cross-process command trace context; verified live on a CC-to-agent wake. +- [x] Inventory source, merge, device-storage, and fleet-operation spans. +- [x] Agent connection phase spans and retry events, without entered guards across awaits. +- [x] Terminal snapshot generation, transport attachment, resume, and backpressure instrumentation. +- [x] Separate hostname-resolution and neighbor-table spans; a live slow wake spent about four seconds in resolution. +- [x] Metrics recipes for rates, duration averages/percentiles, terminal activity, and snapshots. +- [x] Authenticated live logs, metrics, and traces confirmed through Grafana's read-only API. + +See [telemetry setup](telemetry.md), [metrics recipes](telemetry-recipes.md), and +[OpenWrt UCI](openwrt-uci.md). Terminal instrumentation has automated coverage; +live browser recovery and shutdown behavior are not established by those checks. + +## Device Management And Presentation + +- [ ] Make **forget remembered device** reachable in the frontend. Verify the existing backend before adding routes: remove the known identity and its identifiers, not agent observations. An observed device may reappear as unknown. +- [ ] Add remembered-device rename support, with backend validation and frontend editing. +- [ ] Check and complete editing of remembered-device notes. +- [ ] Replace generic unnamed-row labels with useful fallbacks. Offline IP-only rows: `Offline device: `; unknown-presence IP-only rows: `Device: `. Use MAC fallback when no IP exists; retain a generic label only when neither exists. +- [ ] Review frontend visibility defaults. Keep an explicit all-rows option; do not hide evidence just to fix poor labels. The user deliberately enabled all-row querying. + +Identifier detachment already exists in the current frontend source; it is **not** +the requested whole-device removal. No automatic identity promotion or deletion +of remembered identifiers based on observation expiry. + +## Telemetry Follow-Ups + +- [x] Separate console, OTLP log, and trace filters using per-layer filtering. Explicit signal config overrides `RUST_LOG`, then CLI verbosity supplies defaults. Unconfigured defaults remain info-level; exporter-failure logs stay local. Live validation pending. +- [ ] Record hostname-resolution success/failure without exporting hostnames or resolved addresses. Current spans show timing but do not explicitly classify the result. +- [ ] Investigate the slow resolver with a reproducible device-side check. Timing isolated the phase, not the cause; do not treat instrumentation as a fix. +- [ ] Propagate trace context for periodic snapshots so agent collection and CC persistence can be inspected together. Commands already have this linkage. +- [ ] Add current-state device gauges: stored rows per agent and grouped fleet devices split known/unknown. Define scope and update lifecycle first; never derive gauges from the last filtered UI query. +- [ ] Decide whether snapshots need persistence acknowledgements. The existing sent-device counter does not prove CC storage; any confirmed-storage metric needs an authoritative measurement point. + +The existing snapshot counter is cumulative collection volume, not unique devices +or current fleet size. Metric labels should remain bounded; no per-device labels. + +## Validation And Reliability + +- [ ] Reproduce/check Ctrl-C cleanup on the test device: process groups, PTYs, relay tasks, and telemetry shutdown. Leftovers were suspected, not confirmed. +- [ ] Exercise collector unavailability and recovery with running agent/CC instances. Verify normal operations continue and telemetry queues remain bounded; outage losses are expected under the current best-effort policy. +- [ ] Validate terminal recovery instrumentation live: generation timing/size, transport attach/resume, and backpressure episodes ending in drain or disconnect. +- [ ] Validate the revised UCI credential prompt and service environment on the live router. + +## Watch, Not Yet Fix + +- Fleet listing time is mostly device-store reads in observed traces. Measure + sustained latency and row counts before changing queries or indexes. +- Snapshot write durations vary. Existing store spans include pool/lock waits, + transaction work, and result assembly where applicable; they are not pure SQL timings. + +## Deferred + +- Pinned-state editing: currently not needed for the small remembered fleet. +- Browser-render acknowledgement: attach acceptance and snapshot generation do + not prove xterm finished restoring. This would require additional protocol work. +- Retaining terminal images across recovery: text-screen snapshots do not provide image replay. +- CPU/heap profiling: use targeted profiling when a measured issue calls for it; + OTLP traces are not CPU profiles or flame graphs. +- Grafana dashboard construction is the user's exercise; keep improving the recipe book rather than taking over dashboard editing. + +## Release Notes + +- No version bump is made by this roadmap. Additive optional config fields alone + do not require 0.4.0; reconsider if compatibility or defaults change materially. +- Recheck live telemetry after deployment. A Grafana volume reset can remove + history and invalidate service-account tokens or data-source UIDs. +- Never commit Grafana tokens, UCI credentials, or router backups. diff --git a/docs/telemetry.md b/docs/telemetry.md index 53139f6..e1afee1 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -72,8 +72,20 @@ once after installing the updated unit. OpenWrt's HUP reload does not reload the process environment. For a foreground WSL test, export the variable in that shell and use `http://127.0.0.1:4317` with the SSH tunnel; do not put credentials in CLI args. -`RUST_LOG` overrides the verbosity defaults for both local output and exports. -For example, `RUST_LOG=warn` suppresses informational command spans as well as logs. +Console logs, OTLP logs, and traces can have independent filters: + +```toml +[telemetry] +console_filter = "info" +otlp_log_filter = "debug" +trace_filter = "info" +``` + +For each signal, an explicit filter wins over `RUST_LOG`, which wins over CLI +verbosity (`-v`). With no filters, environment override, or verbosity flags, the +application defaults remain info-level with SDK warnings. Metrics are unaffected. +Explicit filters must be valid and nonempty; changing them requires a restart. +Without explicit filters, `RUST_LOG=warn` suppresses informational spans and logs. Use `wakey_agent=info,wakey=info,wakey_core=info,opentelemetry=warn` for the agent and `wakey_control_plane=info,wakey_core=info,opentelemetry=warn` for CC. Exporter warnings remain local to prevent export-failure feedback loops. Increasing verbosity exports the diff --git a/wakey-agent/src/config.rs b/wakey-agent/src/config.rs index a68cb5e..6ebbe8e 100644 --- a/wakey-agent/src/config.rs +++ b/wakey-agent/src/config.rs @@ -283,6 +283,9 @@ mod tests { authorization_env: Some("WAKEY_OTLP_AUTHORIZATION".into()), service_name: "router-agent".into(), json_logs: true, + console_filter: Some("info".into()), + otlp_log_filter: Some("debug".into()), + trace_filter: Some("info".into()), }, terminal: TerminalConfig { enabled: true, diff --git a/wakey-agent/src/enroll.rs b/wakey-agent/src/enroll.rs index deb6582..e77b2b6 100644 --- a/wakey-agent/src/enroll.rs +++ b/wakey-agent/src/enroll.rs @@ -156,6 +156,7 @@ mod tests { authorization_env: Some("WAKEY_OTLP_AUTHORIZATION".into()), service_name: "router-agent".into(), json_logs: true, + ..Default::default() }, terminal: crate::config::TerminalConfig { enabled: true, diff --git a/wakey-control-plane/src/config/init.rs b/wakey-control-plane/src/config/init.rs index 18b495a..218edc5 100644 --- a/wakey-control-plane/src/config/init.rs +++ b/wakey-control-plane/src/config/init.rs @@ -113,6 +113,15 @@ pub fn write_init_config(args: &InitConfigArgs) -> Result> { ui_dist_dir, bootstrap_enroll_tokens, telemetry: WritableTelemetry { + console_filter: base_telemetry + .console_filter + .or_else(|| defaults.telemetry.console_filter.clone()), + otlp_log_filter: base_telemetry + .otlp_log_filter + .or_else(|| defaults.telemetry.otlp_log_filter.clone()), + trace_filter: base_telemetry + .trace_filter + .or_else(|| defaults.telemetry.trace_filter.clone()), authorization_env: base_telemetry .authorization_env .or_else(|| defaults.telemetry.authorization_env.clone()), @@ -227,6 +236,9 @@ bootstrap_enroll_tokens = ["old-token"] [telemetry] otlp_endpoint = "http://otel.example" authorization_env = "WAKEY_OTLP_AUTHORIZATION" +console_filter = "info" +otlp_log_filter = "debug" +trace_filter = "info" service_name = "old-service" json_logs = true "#, @@ -265,6 +277,12 @@ json_logs = true Some("/tmp/wakey-base/old.sqlite3") ); assert_eq!(parsed["telemetry"]["json_logs"].as_bool(), Some(true)); + assert_eq!(parsed["telemetry"]["console_filter"].as_str(), Some("info")); + assert_eq!( + parsed["telemetry"]["otlp_log_filter"].as_str(), + Some("debug") + ); + assert_eq!(parsed["telemetry"]["trace_filter"].as_str(), Some("info")); assert_eq!( parsed["telemetry"]["authorization_env"].as_str(), Some("WAKEY_OTLP_AUTHORIZATION") diff --git a/wakey-control-plane/src/config/resolve.rs b/wakey-control-plane/src/config/resolve.rs index 500e633..13ce2ad 100644 --- a/wakey-control-plane/src/config/resolve.rs +++ b/wakey-control-plane/src/config/resolve.rs @@ -232,6 +232,9 @@ fn resolve_telemetry(file: Option) -> TelemetryConfig { let mut out = TelemetryConfig::default(); if let Some(file) = file { out.authorization_env = file.authorization_env; + out.console_filter = file.console_filter; + out.otlp_log_filter = file.otlp_log_filter; + out.trace_filter = file.trace_filter; if let Some(endpoint) = file.otlp_endpoint { let trimmed = endpoint.trim().to_string(); if !trimmed.is_empty() { diff --git a/wakey-control-plane/src/config/types.rs b/wakey-control-plane/src/config/types.rs index 2396c2f..8d970dd 100644 --- a/wakey-control-plane/src/config/types.rs +++ b/wakey-control-plane/src/config/types.rs @@ -22,6 +22,9 @@ pub struct DaemonConfig { #[derive(Debug, Clone)] pub struct TelemetryConfig { + pub console_filter: Option, + pub otlp_log_filter: Option, + pub trace_filter: Option, pub otlp_endpoint: Option, pub authorization_env: Option, pub service_name: String, @@ -32,6 +35,9 @@ impl Default for TelemetryConfig { fn default() -> Self { let defaults = &DEFAULT_CONFIG.telemetry; Self { + console_filter: defaults.console_filter.clone(), + otlp_log_filter: defaults.otlp_log_filter.clone(), + trace_filter: defaults.trace_filter.clone(), otlp_endpoint: defaults.otlp_endpoint.clone(), authorization_env: defaults.authorization_env.clone(), service_name: defaults.service_name.clone(), @@ -58,6 +64,9 @@ pub(crate) struct FileConfig { #[derive(Debug, Deserialize, Default)] pub(crate) struct FileTelemetryConfig { + pub(crate) console_filter: Option, + pub(crate) otlp_log_filter: Option, + pub(crate) trace_filter: Option, pub(crate) otlp_endpoint: Option, pub(crate) authorization_env: Option, pub(crate) service_name: Option, @@ -82,6 +91,12 @@ pub(crate) struct WritableConfig { #[derive(Debug, Clone, Serialize)] pub(crate) struct WritableTelemetry { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) console_filter: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) otlp_log_filter: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) trace_filter: Option, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) otlp_endpoint: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -102,6 +117,9 @@ pub(crate) static DEFAULT_CONFIG: LazyLock = LazyLock::new(|| Wr ui_dist_dir: "ui/dist".into(), bootstrap_enroll_tokens: Vec::new(), telemetry: WritableTelemetry { + console_filter: None, + otlp_log_filter: None, + trace_filter: None, otlp_endpoint: None, authorization_env: None, service_name: "wakey-control-plane".to_string(), diff --git a/wakey-control-plane/src/tracing.rs b/wakey-control-plane/src/tracing.rs index 29e6c17..8ffafcd 100644 --- a/wakey-control-plane/src/tracing.rs +++ b/wakey-control-plane/src/tracing.rs @@ -8,6 +8,9 @@ pub fn init(verbose: u8, telemetry: &TelemetryConfig) -> Result authorization_env: telemetry.authorization_env.clone(), service_name: telemetry.service_name.clone(), json_logs: telemetry.json_logs, + console_filter: telemetry.console_filter.clone(), + otlp_log_filter: telemetry.otlp_log_filter.clone(), + trace_filter: telemetry.trace_filter.clone(), }; wakey_core::telemetry::init( &config, diff --git a/wakey-core/src/telemetry.rs b/wakey-core/src/telemetry.rs index b666294..4256a45 100644 --- a/wakey-core/src/telemetry.rs +++ b/wakey-core/src/telemetry.rs @@ -28,6 +28,13 @@ pub struct TelemetryConfig { pub authorization_env: Option, pub service_name: String, pub json_logs: bool, + /// Explicit per-signal filters override RUST_LOG and CLI verbosity. + #[serde(skip_serializing_if = "Option::is_none")] + pub console_filter: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub otlp_log_filter: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_filter: Option, } /// Owns providers until all instrumented work has stopped; flush off the async workers. @@ -59,8 +66,12 @@ pub fn init( default_filter: &str, instance: Option<&str>, ) -> Result { - let filter = - EnvFilter::try_from_default_env().or_else(|_| EnvFilter::try_new(default_filter))?; + let rust_log = std::env::var("RUST_LOG").ok(); + let console_filter = signal_filter( + config.console_filter.as_deref(), + rust_log.as_deref(), + default_filter, + )?; let service = if config.service_name.is_empty() { service } else { @@ -72,6 +83,18 @@ pub fn init( .as_deref() .filter(|s| !s.trim().is_empty()) { + // Filters belong to individual layers: a quiet console must not globally + // disable debug events needed by the OTLP log exporter. + let trace_filter = signal_filter( + config.trace_filter.as_deref(), + rust_log.as_deref(), + default_filter, + )?; + let log_filter = signal_filter( + config.otlp_log_filter.as_deref(), + rust_log.as_deref(), + default_filter, + )?; let metadata = authorization_metadata(config.authorization_env.as_deref(), |name| { std::env::var(name).ok() })?; @@ -137,24 +160,33 @@ pub fn init( global::set_tracer_provider(traces.clone()); global::set_meter_provider(metrics.clone()); let _ = METRICS.set(Instruments::new()); - let otel = tracing_opentelemetry::layer().with_tracer(traces.tracer(service.to_owned())); + let otel = tracing_opentelemetry::layer() + .with_tracer(traces.tracer(service.to_owned())) + .with_filter(trace_filter); // Exporter failures must remain local, never feed back into the failing exporter. let logs_layer = opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&logs) + .with_filter(log_filter) .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { !metadata.target().starts_with("opentelemetry") })); - let subscriber = tracing_subscriber::registry() - .with(filter) - .with(otel) - .with(logs_layer); + let subscriber = tracing_subscriber::registry().with(otel).with(logs_layer); if config.json_logs { subscriber - .with(fmt::layer().json().with_timer(local_timer())) + .with( + fmt::layer() + .json() + .with_timer(local_timer()) + .with_filter(console_filter), + ) .try_init()?; } else { subscriber - .with(fmt::layer().with_timer(local_timer())) + .with( + fmt::layer() + .with_timer(local_timer()) + .with_filter(console_filter), + ) .try_init()?; } guard = TelemetryGuard { @@ -164,18 +196,46 @@ pub fn init( }; } else if config.json_logs { tracing_subscriber::registry() - .with(filter) - .with(fmt::layer().json().with_timer(local_timer())) + .with( + fmt::layer() + .json() + .with_timer(local_timer()) + .with_filter(console_filter), + ) .try_init()?; } else { tracing_subscriber::registry() - .with(filter) - .with(fmt::layer().with_timer(local_timer())) + .with( + fmt::layer() + .with_timer(local_timer()) + .with_filter(console_filter), + ) .try_init()?; } Ok(guard) } +fn signal_filter( + explicit: Option<&str>, + rust_log: Option<&str>, + default_filter: &str, +) -> Result { + if let Some(value) = explicit { + anyhow::ensure!( + !value.trim().is_empty(), + "telemetry filter must not be empty" + ); + return EnvFilter::try_new(value).context("invalid telemetry filter"); + } + // Preserve the existing fallback for an absent or invalid RUST_LOG. + rust_log + .and_then(|value| EnvFilter::try_new(value).ok()) + .map(Ok) + .unwrap_or_else(|| { + EnvFilter::try_new(default_filter).context("invalid default telemetry filter") + }) +} + fn authorization_metadata( name: Option<&str>, lookup: impl FnOnce(&str) -> Option, @@ -364,6 +424,30 @@ impl Drop for ActiveTerminal { #[cfg(test)] mod auth_tests { + #[test] + fn explicit_signal_filters_override_environment_and_validate() { + assert_eq!( + super::signal_filter(Some("debug"), Some("warn"), "info") + .unwrap() + .to_string(), + "debug" + ); + assert_eq!( + super::signal_filter(None, Some("warn"), "info") + .unwrap() + .to_string(), + "warn" + ); + assert_eq!( + super::signal_filter(None, None, "info") + .unwrap() + .to_string(), + "info" + ); + assert!(super::signal_filter(Some(""), None, "info").is_err()); + assert!(super::signal_filter(Some("["), None, "info").is_err()); + } + use super::*; #[test] diff --git a/wakey-core/tests/telemetry.rs b/wakey-core/tests/telemetry.rs index 125e5c5..61619b5 100644 --- a/wakey-core/tests/telemetry.rs +++ b/wakey-core/tests/telemetry.rs @@ -68,7 +68,7 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { // Exercise the real global subscriber without mutating this process's environment // or inheriting a developer's RUST_LOG=warn/off setting. if std::env::var_os("WAKEY_OTLP_TEST_CHILD").is_none() { - let status = std::process::Command::new(std::env::current_exe().unwrap()) + let output = std::process::Command::new(std::env::current_exe().unwrap()) .args([ "--exact", "exports_all_signals_and_preserves_remote_command_parent", @@ -77,9 +77,19 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { .env("WAKEY_OTLP_TEST_CHILD", "1") .env("RUST_LOG", "info") .env("WAKEY_OTLP_TEST_AUTH", "Basic dGVzdDp0ZXN0") - .status() + .output() .unwrap(); - assert!(status.success()); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + for console in [&output.stdout, &output.stderr] { + assert!( + !String::from_utf8_lossy(console) + .contains("debug log exported without console output") + ); + } return; } let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -109,6 +119,9 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { authorization_env: Some("WAKEY_OTLP_TEST_AUTH".into()), service_name: "wakey-test".into(), json_logs: false, + console_filter: Some("info".into()), + otlp_log_filter: Some("debug".into()), + trace_filter: Some("info".into()), }, "unused", "info", @@ -130,6 +143,9 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { }); drop(child); drop(parent); + tracing::debug!("debug log exported without console output"); + let hidden = tracing::debug_span!("debug_span_not_exported"); + drop(hidden); tracing::warn!(target: "opentelemetry_sdk", "synthetic exporter failure remains local"); let invalid_parent = tracing::info_span!("invalid_remote_parent"); telemetry::set_remote_parent( @@ -161,6 +177,11 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { assert!(invalid_parent.parent_span_id.is_empty()); assert_ne!(parent.trace_id, invalid_parent.trace_id); let logs = collector.logs.lock().unwrap(); + assert!( + !spans + .iter() + .any(|span| span.name == "debug_span_not_exported") + ); let records: Vec<_> = logs .iter() .flat_map(|r| &r.resource_logs) @@ -177,6 +198,11 @@ async fn exports_all_signals_and_preserves_remote_command_parent() { .iter() .any(|r| format!("{:?}", r.body).contains("synthetic exporter failure")) ); + assert!( + records + .iter() + .any(|r| format!("{:?}", r.body).contains("debug log exported without console output")) + ); let metrics = collector.metrics.lock().unwrap(); let names: Vec<_> = metrics .iter()