diff --git a/api/pom.xml b/api/pom.xml
index d5791bed38e6..d4f9e3d10083 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -71,6 +71,10 @@
cloud-framework-direct-download
${project.version}
+
+ io.opentelemetry
+ opentelemetry-api
+
diff --git a/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java
new file mode 100644
index 000000000000..f67cf3f7ced1
--- /dev/null
+++ b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.api.filter;
+
+import org.apache.cloudstack.context.LogContext;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.util.UUID;
+import javax.servlet.FilterConfig;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+
+public class ApiTraceFilter implements Filter {
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+ throws IOException, ServletException {
+ try {
+ HttpServletRequest httpReq = (HttpServletRequest) request;
+ String traceId = httpReq.getHeader(LogContext.X_B3_TRACEID_KEY);
+ if (StringUtils.isBlank(traceId)) {
+ traceId = UUID.randomUUID().toString();
+ }
+
+ LogContext.current().putContextParameter(LogContext.X_B3_TRACEID_KEY, traceId);
+ chain.doFilter(request, response);
+ } finally {
+ LogContext.current().removeContextParameter(LogContext.X_B3_TRACEID_KEY);
+ }
+ }
+
+ @Override
+ public void destroy() {
+ }
+}
diff --git a/api/src/main/java/org/apache/cloudstack/context/LogContext.java b/api/src/main/java/org/apache/cloudstack/context/LogContext.java
index c367975aba3b..e63022f40af2 100644
--- a/api/src/main/java/org/apache/cloudstack/context/LogContext.java
+++ b/api/src/main/java/org/apache/cloudstack/context/LogContext.java
@@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.context;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -53,6 +54,10 @@ public class LogContext {
private long userId;
private final Map context = new HashMap();
+ public final static String X_B3_TRACEID_KEY = "traceid";
+ public final static String MOSAIC_TRACE_ID_KEY = "mosaic_trace_id";
+ public final static String MOSAIC_SPAN_ID_KEY = "mosaic_span_id";
+
static EntityManager s_entityMgr;
public static void init(EntityManager entityMgr) {
@@ -78,6 +83,20 @@ protected LogContext(User user, Account account, String logContextId) {
public void putContextParameter(String key, String value) {
context.put(key, value);
+ MDC.put(key, value);
+ }
+
+ public void removeContextParameter(String key) {
+ context.remove(key);
+ MDC.remove(key);
+ }
+
+ public void removeContextParameters() {
+ // Iterate over a copy of the keys: removeContextParameter mutates the context
+ // map, so iterating the live keySet/entrySet would throw ConcurrentModificationException.
+ for (String key : new ArrayList<>(context.keySet())) {
+ removeContextParameter(key);
+ }
}
public String getContextParameter(String key) {
diff --git a/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java
new file mode 100644
index 000000000000..2b44fe84e260
--- /dev/null
+++ b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import org.apache.log4j.MDC;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+
+/**
+ * Mirrors the active OpenTelemetry span onto the Log4j MDC so management-server log
+ * lines carry mosaic_trace_id and mosaic_span_id on every thread that has an active
+ * span (API requests, agent-command dispatch, async jobs), not just the servlet path.
+ *
+ * The OpenTelemetry agent populates the log MDC automatically for Log4j2 and Logback,
+ * but not for Log4j 1.2 (reload4j), which the management server uses. This wrapper
+ * fills that gap by hooking the OpenTelemetry context lifecycle: whenever a span
+ * becomes current on a thread it copies the ids into the MDC, and restores the
+ * previous values when that scope closes. Install once at startup via {@link #register()}.
+ */
+public class TraceContextMdcWrapper implements ContextStorage {
+
+ private final ContextStorage delegate;
+
+ TraceContextMdcWrapper(ContextStorage delegate) {
+ this.delegate = delegate;
+ }
+
+ /**
+ * Install the wrapper. Must be called before the first OpenTelemetry context is
+ * used, i.e. at management-server startup, before the server accepts requests.
+ */
+ public static void register() {
+ ContextStorage.addWrapper(TraceContextMdcWrapper::new);
+ }
+
+ @Override
+ public Scope attach(Context toAttach) {
+ Object previousTraceId = MDC.get(LogContext.MOSAIC_TRACE_ID_KEY);
+ Object previousSpanId = MDC.get(LogContext.MOSAIC_SPAN_ID_KEY);
+ SpanContext spanContext = Span.fromContext(toAttach).getSpanContext();
+ if (spanContext.isValid()) {
+ MDC.put(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId());
+ MDC.put(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId());
+ } else {
+ MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY);
+ MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY);
+ }
+ Scope delegateScope = delegate.attach(toAttach);
+ return () -> {
+ delegateScope.close();
+ restore(LogContext.MOSAIC_TRACE_ID_KEY, previousTraceId);
+ restore(LogContext.MOSAIC_SPAN_ID_KEY, previousSpanId);
+ };
+ }
+
+ private static void restore(String key, Object previous) {
+ if (previous != null) {
+ MDC.put(key, previous);
+ } else {
+ MDC.remove(key);
+ }
+ }
+
+ @Override
+ public Context current() {
+ return delegate.current();
+ }
+}
diff --git a/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java
new file mode 100644
index 000000000000..b1ad9dab3399
--- /dev/null
+++ b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import org.apache.log4j.MDC;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.TraceFlags;
+import io.opentelemetry.api.trace.TraceState;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+
+public class TraceContextMdcWrapperTest {
+
+ private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";
+ private static final String SPAN_ID = "00f067aa0ba902b7";
+ private static final String OTHER_TRACE_ID = "d75597dcda4f6e7b9c1a2b3c4d5e6f70";
+ private static final String OTHER_SPAN_ID = "aabbccddeeff0011";
+
+ // Minimal delegate so we test the wrapper in isolation, no real context storage.
+ private final ContextStorage noopDelegate = new ContextStorage() {
+ @Override
+ public Scope attach(Context toAttach) {
+ return () -> { };
+ }
+
+ @Override
+ public Context current() {
+ return Context.root();
+ }
+ };
+
+ private final TraceContextMdcWrapper wrapper = new TraceContextMdcWrapper(noopDelegate);
+
+ @After
+ public void tearDown() {
+ MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY);
+ MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY);
+ }
+
+ private static Context contextWithSpan(String traceId, String spanId) {
+ return Context.root().with(Span.wrap(
+ SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault())));
+ }
+
+ @Test
+ public void putsTraceContextOnMdcWhileScopeOpenAndRestoresOnClose() {
+ Scope scope = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+ Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+ Assert.assertEquals(SPAN_ID, MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+
+ scope.close();
+ Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+ Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+ }
+
+ @Test
+ public void leavesMdcUnsetWhenNoActiveSpan() {
+ Scope scope = wrapper.attach(Context.root());
+ Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+ Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+ scope.close();
+ }
+
+ @Test
+ public void restoresOuterSpanWhenNestedScopeCloses() {
+ Scope outer = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+ Scope inner = wrapper.attach(contextWithSpan(OTHER_TRACE_ID, OTHER_SPAN_ID));
+ Assert.assertEquals(OTHER_TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+
+ inner.close();
+ Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+
+ outer.close();
+ Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+ }
+}
diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java
index 06477fff8986..4cdccdf51ba6 100644
--- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java
+++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java
@@ -55,6 +55,8 @@
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.server.ServerProperties;
+import org.apache.cloudstack.context.TraceContextMdcWrapper;
+
/***
* The ServerDaemon class implements the embedded server, it can be started either
* using JSVC or directly from the JAR along with additional jars not shaded in the uber-jar.
@@ -108,6 +110,9 @@ public class ServerDaemon implements Daemon {
//////////////////////////////////////////////////
public static void main(final String... anArgs) throws Exception {
+ // Install the trace-context to MDC hook before the server starts, so every
+ // thread with an active OpenTelemetry span carries mosaic_trace_id in its logs.
+ TraceContextMdcWrapper.register();
final ServerDaemon daemon = new ServerDaemon();
daemon.init(null);
daemon.start();
diff --git a/client/src/main/webapp/WEB-INF/web.xml b/client/src/main/webapp/WEB-INF/web.xml
index 43bee7e59d88..fdb899b55562 100644
--- a/client/src/main/webapp/WEB-INF/web.xml
+++ b/client/src/main/webapp/WEB-INF/web.xml
@@ -36,6 +36,16 @@
classpath:META-INF/cloudstack/webApplicationContext.xml
+
+ apiTraceFilter
+ org.apache.cloudstack.api.filter.ApiTraceFilter
+
+
+
+ apiTraceFilter
+ /api/*
+
+
cloudStartupServlet
com.cloud.servlet.CloudStartupServlet