From 680e0f91210480a0aa5ff161e3fa643efcabab9a Mon Sep 17 00:00:00 2001 From: Ashley Ou Date: Wed, 22 Jul 2026 16:23:26 -0700 Subject: [PATCH 1/2] api: add ApiTraceFilter with request trace id and active OpenTelemetry span trace id and span id --- api/pom.xml | 5 + .../cloudstack/api/filter/ApiTraceFilter.java | 73 +++++++++++++++ .../apache/cloudstack/context/LogContext.java | 16 ++++ .../api/filter/ApiTraceFilterTest.java | 92 +++++++++++++++++++ client/src/main/webapp/WEB-INF/web.xml | 10 ++ 5 files changed, 196 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java diff --git a/api/pom.xml b/api/pom.xml index d5791bed38e6..27eda410514f 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -71,6 +71,11 @@ cloud-framework-direct-download ${project.version} + + io.opentelemetry + opentelemetry-api + 1.51.0 + diff --git a/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java new file mode 100644 index 000000000000..f384bab11a26 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.filter; + +import org.apache.cloudstack.context.LogContext; +import java.io.IOException; +import java.util.UUID; +import javax.servlet.FilterConfig; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; + +public class ApiTraceFilter implements Filter { + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + // Also record the active OpenTelemetry span so log lines can be joined to the + // distributed trace. Sourced from the span, not a header, so no id is invented; + // when there is no valid span the keys are left unset and render empty. + SpanContext spanContext = Span.current().getSpanContext(); + boolean spanApplied = spanContext.isValid(); + try { + HttpServletRequest httpReq = (HttpServletRequest) request; + String traceId = httpReq.getHeader(LogContext.X_B3_TRACEID_KEY); + if (traceId == null || traceId.isEmpty()) { + traceId = UUID.randomUUID().toString(); + } + + LogContext.current().putContextParameter(LogContext.X_B3_TRACEID_KEY, traceId); + if (spanApplied) { + LogContext.current().putContextParameter(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId()); + LogContext.current().putContextParameter(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId()); + } + chain.doFilter(request, response); + } finally { + LogContext.current().removeContextParameter(LogContext.X_B3_TRACEID_KEY); + if (spanApplied) { + LogContext.current().removeContextParameter(LogContext.MOSAIC_TRACE_ID_KEY); + LogContext.current().removeContextParameter(LogContext.MOSAIC_SPAN_ID_KEY); + } + } + } + + @Override + public void destroy() { + } +} diff --git a/api/src/main/java/org/apache/cloudstack/context/LogContext.java b/api/src/main/java/org/apache/cloudstack/context/LogContext.java index c367975aba3b..4828de90faec 100644 --- a/api/src/main/java/org/apache/cloudstack/context/LogContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/LogContext.java @@ -53,6 +53,10 @@ public class LogContext { private long userId; private final Map context = new HashMap(); + public final static String X_B3_TRACEID_KEY = "traceid"; + public final static String MOSAIC_TRACE_ID_KEY = "mosaic_trace_id"; + public final static String MOSAIC_SPAN_ID_KEY = "mosaic_span_id"; + static EntityManager s_entityMgr; public static void init(EntityManager entityMgr) { @@ -78,6 +82,18 @@ protected LogContext(User user, Account account, String logContextId) { public void putContextParameter(String key, String value) { context.put(key, value); + MDC.put(key, value); + } + + public void removeContextParameter(String key) { + context.remove(key); + MDC.remove(key); + } + + public void removeContextParameters() { + for (Map.Entry entry : context.entrySet()) { + removeContextParameter(entry.getKey()); + } } public String getContextParameter(String key) { diff --git a/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java b/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java new file mode 100644 index 000000000000..4d5d8654995e --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.filter; + +import javax.servlet.FilterChain; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import org.apache.cloudstack.context.LogContext; +import org.apache.log4j.MDC; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Scope; + +public class ApiTraceFilterTest { + + private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; + private static final String SPAN_ID = "00f067aa0ba902b7"; + + private final ApiTraceFilter filter = new ApiTraceFilter(); + + @After + public void tearDown() { + LogContext.unregister(); + } + + @Test + public void putsMosaicTraceContextOnMdcWhenSpanActive() throws Exception { + HttpServletRequest req = Mockito.mock(HttpServletRequest.class); + Mockito.when(req.getHeader(LogContext.X_B3_TRACEID_KEY)).thenReturn(null); + String[] duringChain = new String[2]; + FilterChain chain = (rq, rs) -> { + duringChain[0] = (String) MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); + duringChain[1] = (String) MDC.get(LogContext.MOSAIC_SPAN_ID_KEY); + }; + + SpanContext spanContext = + SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getSampled(), TraceState.getDefault()); + try (Scope scope = Span.wrap(spanContext).makeCurrent()) { + filter.doFilter(req, Mockito.mock(ServletResponse.class), chain); + } + + // Present on the MDC while the request is in flight, from the active span. + Assert.assertEquals(TRACE_ID, duringChain[0]); + Assert.assertEquals(SPAN_ID, duringChain[1]); + // Removed once the request completes, so pooled threads do not leak it. + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + } + + @Test + public void leavesMosaicKeysUnsetWhenNoActiveSpan() throws Exception { + HttpServletRequest req = Mockito.mock(HttpServletRequest.class); + Mockito.when(req.getHeader(LogContext.X_B3_TRACEID_KEY)).thenReturn(null); + boolean[] chainInvoked = {false}; + String[] duringChain = new String[1]; + FilterChain chain = (rq, rs) -> { + chainInvoked[0] = true; + duringChain[0] = (String) MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); + }; + + // No span in scope: Span.current() is the invalid default span. + filter.doFilter(req, Mockito.mock(ServletResponse.class), chain); + + Assert.assertTrue(chainInvoked[0]); + Assert.assertNull(duringChain[0]); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + } +} diff --git a/client/src/main/webapp/WEB-INF/web.xml b/client/src/main/webapp/WEB-INF/web.xml index 43bee7e59d88..fdb899b55562 100644 --- a/client/src/main/webapp/WEB-INF/web.xml +++ b/client/src/main/webapp/WEB-INF/web.xml @@ -36,6 +36,16 @@ classpath:META-INF/cloudstack/webApplicationContext.xml + + apiTraceFilter + org.apache.cloudstack.api.filter.ApiTraceFilter + + + + apiTraceFilter + /api/* + + cloudStartupServlet com.cloud.servlet.CloudStartupServlet From 58fcc64109af37de0fdc2cdc4bb1072794008766 Mon Sep 17 00:00:00 2001 From: Ashleyyq Date: Mon, 3 Aug 2026 10:00:24 -0700 Subject: [PATCH 2/2] api: sstamp trace id via a central OpenTelemetry context hook; inherit managed opentelemetry-api version; fix removeContextParameters CME --- api/pom.xml | 1 - .../cloudstack/api/filter/ApiTraceFilter.java | 20 +--- .../apache/cloudstack/context/LogContext.java | 7 +- .../context/TraceContextMdcWrapper.java | 86 +++++++++++++++++ .../api/filter/ApiTraceFilterTest.java | 92 ------------------ .../context/TraceContextMdcWrapperTest.java | 96 +++++++++++++++++++ .../org/apache/cloudstack/ServerDaemon.java | 5 + 7 files changed, 195 insertions(+), 112 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java delete mode 100644 api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java diff --git a/api/pom.xml b/api/pom.xml index 27eda410514f..d4f9e3d10083 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -74,7 +74,6 @@ io.opentelemetry opentelemetry-api - 1.51.0 diff --git a/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java index f384bab11a26..f67cf3f7ced1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java +++ b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java @@ -19,6 +19,8 @@ package org.apache.cloudstack.api.filter; import org.apache.cloudstack.context.LogContext; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.util.UUID; import javax.servlet.FilterConfig; @@ -29,9 +31,6 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; - public class ApiTraceFilter implements Filter { @Override public void init(FilterConfig filterConfig) throws ServletException { @@ -40,30 +39,17 @@ public void init(FilterConfig filterConfig) throws ServletException { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - // Also record the active OpenTelemetry span so log lines can be joined to the - // distributed trace. Sourced from the span, not a header, so no id is invented; - // when there is no valid span the keys are left unset and render empty. - SpanContext spanContext = Span.current().getSpanContext(); - boolean spanApplied = spanContext.isValid(); try { HttpServletRequest httpReq = (HttpServletRequest) request; String traceId = httpReq.getHeader(LogContext.X_B3_TRACEID_KEY); - if (traceId == null || traceId.isEmpty()) { + if (StringUtils.isBlank(traceId)) { traceId = UUID.randomUUID().toString(); } LogContext.current().putContextParameter(LogContext.X_B3_TRACEID_KEY, traceId); - if (spanApplied) { - LogContext.current().putContextParameter(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId()); - LogContext.current().putContextParameter(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId()); - } chain.doFilter(request, response); } finally { LogContext.current().removeContextParameter(LogContext.X_B3_TRACEID_KEY); - if (spanApplied) { - LogContext.current().removeContextParameter(LogContext.MOSAIC_TRACE_ID_KEY); - LogContext.current().removeContextParameter(LogContext.MOSAIC_SPAN_ID_KEY); - } } } diff --git a/api/src/main/java/org/apache/cloudstack/context/LogContext.java b/api/src/main/java/org/apache/cloudstack/context/LogContext.java index 4828de90faec..e63022f40af2 100644 --- a/api/src/main/java/org/apache/cloudstack/context/LogContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/LogContext.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.context; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -91,8 +92,10 @@ public void removeContextParameter(String key) { } public void removeContextParameters() { - for (Map.Entry entry : context.entrySet()) { - removeContextParameter(entry.getKey()); + // Iterate over a copy of the keys: removeContextParameter mutates the context + // map, so iterating the live keySet/entrySet would throw ConcurrentModificationException. + for (String key : new ArrayList<>(context.keySet())) { + removeContextParameter(key); } } diff --git a/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java new file mode 100644 index 000000000000..2b44fe84e260 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.context; + +import org.apache.log4j.MDC; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; + +/** + * Mirrors the active OpenTelemetry span onto the Log4j MDC so management-server log + * lines carry mosaic_trace_id and mosaic_span_id on every thread that has an active + * span (API requests, agent-command dispatch, async jobs), not just the servlet path. + * + * The OpenTelemetry agent populates the log MDC automatically for Log4j2 and Logback, + * but not for Log4j 1.2 (reload4j), which the management server uses. This wrapper + * fills that gap by hooking the OpenTelemetry context lifecycle: whenever a span + * becomes current on a thread it copies the ids into the MDC, and restores the + * previous values when that scope closes. Install once at startup via {@link #register()}. + */ +public class TraceContextMdcWrapper implements ContextStorage { + + private final ContextStorage delegate; + + TraceContextMdcWrapper(ContextStorage delegate) { + this.delegate = delegate; + } + + /** + * Install the wrapper. Must be called before the first OpenTelemetry context is + * used, i.e. at management-server startup, before the server accepts requests. + */ + public static void register() { + ContextStorage.addWrapper(TraceContextMdcWrapper::new); + } + + @Override + public Scope attach(Context toAttach) { + Object previousTraceId = MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); + Object previousSpanId = MDC.get(LogContext.MOSAIC_SPAN_ID_KEY); + SpanContext spanContext = Span.fromContext(toAttach).getSpanContext(); + if (spanContext.isValid()) { + MDC.put(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId()); + MDC.put(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId()); + } else { + MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY); + MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY); + } + Scope delegateScope = delegate.attach(toAttach); + return () -> { + delegateScope.close(); + restore(LogContext.MOSAIC_TRACE_ID_KEY, previousTraceId); + restore(LogContext.MOSAIC_SPAN_ID_KEY, previousSpanId); + }; + } + + private static void restore(String key, Object previous) { + if (previous != null) { + MDC.put(key, previous); + } else { + MDC.remove(key); + } + } + + @Override + public Context current() { + return delegate.current(); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java b/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java deleted file mode 100644 index 4d5d8654995e..000000000000 --- a/api/src/test/java/org/apache/cloudstack/api/filter/ApiTraceFilterTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.api.filter; - -import javax.servlet.FilterChain; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; - -import org.apache.cloudstack.context.LogContext; -import org.apache.log4j.MDC; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.context.Scope; - -public class ApiTraceFilterTest { - - private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; - private static final String SPAN_ID = "00f067aa0ba902b7"; - - private final ApiTraceFilter filter = new ApiTraceFilter(); - - @After - public void tearDown() { - LogContext.unregister(); - } - - @Test - public void putsMosaicTraceContextOnMdcWhenSpanActive() throws Exception { - HttpServletRequest req = Mockito.mock(HttpServletRequest.class); - Mockito.when(req.getHeader(LogContext.X_B3_TRACEID_KEY)).thenReturn(null); - String[] duringChain = new String[2]; - FilterChain chain = (rq, rs) -> { - duringChain[0] = (String) MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); - duringChain[1] = (String) MDC.get(LogContext.MOSAIC_SPAN_ID_KEY); - }; - - SpanContext spanContext = - SpanContext.create(TRACE_ID, SPAN_ID, TraceFlags.getSampled(), TraceState.getDefault()); - try (Scope scope = Span.wrap(spanContext).makeCurrent()) { - filter.doFilter(req, Mockito.mock(ServletResponse.class), chain); - } - - // Present on the MDC while the request is in flight, from the active span. - Assert.assertEquals(TRACE_ID, duringChain[0]); - Assert.assertEquals(SPAN_ID, duringChain[1]); - // Removed once the request completes, so pooled threads do not leak it. - Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); - Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); - } - - @Test - public void leavesMosaicKeysUnsetWhenNoActiveSpan() throws Exception { - HttpServletRequest req = Mockito.mock(HttpServletRequest.class); - Mockito.when(req.getHeader(LogContext.X_B3_TRACEID_KEY)).thenReturn(null); - boolean[] chainInvoked = {false}; - String[] duringChain = new String[1]; - FilterChain chain = (rq, rs) -> { - chainInvoked[0] = true; - duringChain[0] = (String) MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); - }; - - // No span in scope: Span.current() is the invalid default span. - filter.doFilter(req, Mockito.mock(ServletResponse.class), chain); - - Assert.assertTrue(chainInvoked[0]); - Assert.assertNull(duringChain[0]); - Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); - } -} diff --git a/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java new file mode 100644 index 000000000000..b1ad9dab3399 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.context; + +import org.apache.log4j.MDC; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; + +public class TraceContextMdcWrapperTest { + + private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; + private static final String SPAN_ID = "00f067aa0ba902b7"; + private static final String OTHER_TRACE_ID = "d75597dcda4f6e7b9c1a2b3c4d5e6f70"; + private static final String OTHER_SPAN_ID = "aabbccddeeff0011"; + + // Minimal delegate so we test the wrapper in isolation, no real context storage. + private final ContextStorage noopDelegate = new ContextStorage() { + @Override + public Scope attach(Context toAttach) { + return () -> { }; + } + + @Override + public Context current() { + return Context.root(); + } + }; + + private final TraceContextMdcWrapper wrapper = new TraceContextMdcWrapper(noopDelegate); + + @After + public void tearDown() { + MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY); + MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY); + } + + private static Context contextWithSpan(String traceId, String spanId) { + return Context.root().with(Span.wrap( + SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault()))); + } + + @Test + public void putsTraceContextOnMdcWhileScopeOpenAndRestoresOnClose() { + Scope scope = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID)); + Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertEquals(SPAN_ID, MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + + scope.close(); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + } + + @Test + public void leavesMdcUnsetWhenNoActiveSpan() { + Scope scope = wrapper.attach(Context.root()); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + scope.close(); + } + + @Test + public void restoresOuterSpanWhenNestedScopeCloses() { + Scope outer = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID)); + Scope inner = wrapper.attach(contextWithSpan(OTHER_TRACE_ID, OTHER_SPAN_ID)); + Assert.assertEquals(OTHER_TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + + inner.close(); + Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + + outer.close(); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + } +} diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 06477fff8986..4cdccdf51ba6 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -55,6 +55,8 @@ import com.cloud.utils.PropertiesUtil; import com.cloud.utils.server.ServerProperties; +import org.apache.cloudstack.context.TraceContextMdcWrapper; + /*** * The ServerDaemon class implements the embedded server, it can be started either * using JSVC or directly from the JAR along with additional jars not shaded in the uber-jar. @@ -108,6 +110,9 @@ public class ServerDaemon implements Daemon { ////////////////////////////////////////////////// public static void main(final String... anArgs) throws Exception { + // Install the trace-context to MDC hook before the server starts, so every + // thread with an active OpenTelemetry span carries mosaic_trace_id in its logs. + TraceContextMdcWrapper.register(); final ServerDaemon daemon = new ServerDaemon(); daemon.init(null); daemon.start();