fix: keep exporter diagnostics local and classify rename failures
This commit is contained in:
4 files changed
+174
-9
No files matched your search
@@ -181,14 +181,14 @@ pub fn init(
|
||||
let _ = METRICS.set(Instruments::new());
|
||||
let otel = tracing_opentelemetry::layer()
|
||||
.with_tracer(traces.tracer(service.to_owned()))
|
||||
.with_filter(trace_filter);
|
||||
// Exporter failures must remain local, never feed back into the failing exporter.
|
||||
.with_filter(trace_filter)
|
||||
.with_filter(tracing_subscriber::filter::filter_fn(not_exporter_internal));
|
||||
// Apply the same exclusion to both OTLP layers. Console filtering stays
|
||||
// independent so exporter and transport diagnostics remain available.
|
||||
let logs_layer =
|
||||
opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&logs)
|
||||
.with_filter(log_filter)
|
||||
.with_filter(tracing_subscriber::filter::filter_fn(|metadata| {
|
||||
!metadata.target().starts_with("opentelemetry")
|
||||
}));
|
||||
.with_filter(tracing_subscriber::filter::filter_fn(not_exporter_internal));
|
||||
let subscriber = tracing_subscriber::registry().with(otel).with(logs_layer);
|
||||
if config.json_logs {
|
||||
subscriber
|
||||
@@ -235,6 +235,17 @@ pub fn init(
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
/// Exporting the exporter's own diagnostics can create a perpetual feedback
|
||||
/// loop. Transport targets are shared with application traffic, so exclude
|
||||
/// them from OTLP altogether while retaining console diagnostics.
|
||||
fn not_exporter_internal(metadata: &tracing::Metadata<'_>) -> bool {
|
||||
let target = metadata.target();
|
||||
let root = target.split("::").next().unwrap_or(target);
|
||||
!(root == "opentelemetry"
|
||||
|| root.starts_with("opentelemetry_")
|
||||
|| matches!(root, "h2" | "hyper" | "hyper_util" | "tonic" | "tower"))
|
||||
}
|
||||
|
||||
fn signal_filter(
|
||||
explicit: Option<&str>,
|
||||
rust_log: Option<&str>,
|
||||
|
||||
@@ -238,7 +238,7 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
|
||||
json_logs: false,
|
||||
console_filter: Some("info".into()),
|
||||
otlp_log_filter: Some("debug".into()),
|
||||
trace_filter: Some("info".into()),
|
||||
trace_filter: Some("info,opentelemetry_sdk=debug,h2=debug,hyper=debug,hyper_util=debug,tonic=debug,tower=debug".into()),
|
||||
},
|
||||
"unused",
|
||||
"info",
|
||||
@@ -295,6 +295,20 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
|
||||
let hidden = tracing::debug_span!("debug_span_not_exported");
|
||||
drop(hidden);
|
||||
tracing::warn!(target: "opentelemetry_sdk", "synthetic exporter failure remains local");
|
||||
macro_rules! transport_activity {
|
||||
($target:literal) => {{
|
||||
let span = tracing::debug_span!(target: $target, "exporter_transport_span");
|
||||
span.in_scope(|| tracing::debug!(target: $target, "synthetic exporter transport event"));
|
||||
}};
|
||||
}
|
||||
transport_activity!("opentelemetry_sdk");
|
||||
transport_activity!("h2::proto");
|
||||
transport_activity!("hyper::client");
|
||||
transport_activity!("hyper_util::client");
|
||||
transport_activity!("tonic::transport");
|
||||
transport_activity!("tower::buffer");
|
||||
// A similarly named application target is not transport infrastructure.
|
||||
tracing::info!(target: "tower_device", "application target remains exported");
|
||||
let invalid_parent = tracing::info_span!("invalid_remote_parent");
|
||||
telemetry::set_remote_parent(
|
||||
&invalid_parent,
|
||||
@@ -332,6 +346,12 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
|
||||
assert!(invalid_parent.parent_span_id.is_empty());
|
||||
assert_ne!(parent.trace_id, invalid_parent.trace_id);
|
||||
let logs = collector.logs.lock().unwrap();
|
||||
assert!(
|
||||
!spans
|
||||
.iter()
|
||||
.any(|span| span.name == "exporter_transport_span"),
|
||||
"exporter transport spans must not feed back into OTLP"
|
||||
);
|
||||
assert!(
|
||||
!spans
|
||||
.iter()
|
||||
@@ -343,6 +363,17 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
|
||||
.flat_map(|r| &r.scope_logs)
|
||||
.flat_map(|s| &s.log_records)
|
||||
.collect();
|
||||
assert!(
|
||||
!records
|
||||
.iter()
|
||||
.any(|r| format!("{:?}", r.body).contains("synthetic exporter transport event")),
|
||||
"exporter transport events must not feed back into OTLP"
|
||||
);
|
||||
assert!(
|
||||
records
|
||||
.iter()
|
||||
.any(|r| format!("{:?}", r.body).contains("application target remains exported"))
|
||||
);
|
||||
assert!(
|
||||
records
|
||||
.iter()
|
||||
|
||||
Reference in new issue
Block a user