diff --git a/api/pom.xml b/api/pom.xml index d5791bed38e6..d4f9e3d10083 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -71,6 +71,10 @@ cloud-framework-direct-download ${project.version} + + io.opentelemetry + opentelemetry-api + diff --git a/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java new file mode 100644 index 000000000000..f67cf3f7ced1 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.filter; + +import org.apache.cloudstack.context.LogContext; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.UUID; +import javax.servlet.FilterConfig; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; + +public class ApiTraceFilter implements Filter { + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + try { + HttpServletRequest httpReq = (HttpServletRequest) request; + String traceId = httpReq.getHeader(LogContext.X_B3_TRACEID_KEY); + if (StringUtils.isBlank(traceId)) { + traceId = UUID.randomUUID().toString(); + } + + LogContext.current().putContextParameter(LogContext.X_B3_TRACEID_KEY, traceId); + chain.doFilter(request, response); + } finally { + LogContext.current().removeContextParameter(LogContext.X_B3_TRACEID_KEY); + } + } + + @Override + public void destroy() { + } +} diff --git a/api/src/main/java/org/apache/cloudstack/context/LogContext.java b/api/src/main/java/org/apache/cloudstack/context/LogContext.java index c367975aba3b..e63022f40af2 100644 --- a/api/src/main/java/org/apache/cloudstack/context/LogContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/LogContext.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.context; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -53,6 +54,10 @@ public class LogContext { private long userId; private final Map context = new HashMap(); + public final static String X_B3_TRACEID_KEY = "traceid"; + public final static String MOSAIC_TRACE_ID_KEY = "mosaic_trace_id"; + public final static String MOSAIC_SPAN_ID_KEY = "mosaic_span_id"; + static EntityManager s_entityMgr; public static void init(EntityManager entityMgr) { @@ -78,6 +83,20 @@ protected LogContext(User user, Account account, String logContextId) { public void putContextParameter(String key, String value) { context.put(key, value); + MDC.put(key, value); + } + + public void removeContextParameter(String key) { + context.remove(key); + MDC.remove(key); + } + + public void removeContextParameters() { + // Iterate over a copy of the keys: removeContextParameter mutates the context + // map, so iterating the live keySet/entrySet would throw ConcurrentModificationException. + for (String key : new ArrayList<>(context.keySet())) { + removeContextParameter(key); + } } public String getContextParameter(String key) { diff --git a/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java new file mode 100644 index 000000000000..2b44fe84e260 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.context; + +import org.apache.log4j.MDC; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; + +/** + * Mirrors the active OpenTelemetry span onto the Log4j MDC so management-server log + * lines carry mosaic_trace_id and mosaic_span_id on every thread that has an active + * span (API requests, agent-command dispatch, async jobs), not just the servlet path. + * + * The OpenTelemetry agent populates the log MDC automatically for Log4j2 and Logback, + * but not for Log4j 1.2 (reload4j), which the management server uses. This wrapper + * fills that gap by hooking the OpenTelemetry context lifecycle: whenever a span + * becomes current on a thread it copies the ids into the MDC, and restores the + * previous values when that scope closes. Install once at startup via {@link #register()}. + */ +public class TraceContextMdcWrapper implements ContextStorage { + + private final ContextStorage delegate; + + TraceContextMdcWrapper(ContextStorage delegate) { + this.delegate = delegate; + } + + /** + * Install the wrapper. Must be called before the first OpenTelemetry context is + * used, i.e. at management-server startup, before the server accepts requests. + */ + public static void register() { + ContextStorage.addWrapper(TraceContextMdcWrapper::new); + } + + @Override + public Scope attach(Context toAttach) { + Object previousTraceId = MDC.get(LogContext.MOSAIC_TRACE_ID_KEY); + Object previousSpanId = MDC.get(LogContext.MOSAIC_SPAN_ID_KEY); + SpanContext spanContext = Span.fromContext(toAttach).getSpanContext(); + if (spanContext.isValid()) { + MDC.put(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId()); + MDC.put(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId()); + } else { + MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY); + MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY); + } + Scope delegateScope = delegate.attach(toAttach); + return () -> { + delegateScope.close(); + restore(LogContext.MOSAIC_TRACE_ID_KEY, previousTraceId); + restore(LogContext.MOSAIC_SPAN_ID_KEY, previousSpanId); + }; + } + + private static void restore(String key, Object previous) { + if (previous != null) { + MDC.put(key, previous); + } else { + MDC.remove(key); + } + } + + @Override + public Context current() { + return delegate.current(); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java new file mode 100644 index 000000000000..b1ad9dab3399 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.context; + +import org.apache.log4j.MDC; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; + +public class TraceContextMdcWrapperTest { + + private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; + private static final String SPAN_ID = "00f067aa0ba902b7"; + private static final String OTHER_TRACE_ID = "d75597dcda4f6e7b9c1a2b3c4d5e6f70"; + private static final String OTHER_SPAN_ID = "aabbccddeeff0011"; + + // Minimal delegate so we test the wrapper in isolation, no real context storage. + private final ContextStorage noopDelegate = new ContextStorage() { + @Override + public Scope attach(Context toAttach) { + return () -> { }; + } + + @Override + public Context current() { + return Context.root(); + } + }; + + private final TraceContextMdcWrapper wrapper = new TraceContextMdcWrapper(noopDelegate); + + @After + public void tearDown() { + MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY); + MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY); + } + + private static Context contextWithSpan(String traceId, String spanId) { + return Context.root().with(Span.wrap( + SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault()))); + } + + @Test + public void putsTraceContextOnMdcWhileScopeOpenAndRestoresOnClose() { + Scope scope = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID)); + Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertEquals(SPAN_ID, MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + + scope.close(); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + } + + @Test + public void leavesMdcUnsetWhenNoActiveSpan() { + Scope scope = wrapper.attach(Context.root()); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY)); + scope.close(); + } + + @Test + public void restoresOuterSpanWhenNestedScopeCloses() { + Scope outer = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID)); + Scope inner = wrapper.attach(contextWithSpan(OTHER_TRACE_ID, OTHER_SPAN_ID)); + Assert.assertEquals(OTHER_TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + + inner.close(); + Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + + outer.close(); + Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY)); + } +} diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 06477fff8986..4cdccdf51ba6 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -55,6 +55,8 @@ import com.cloud.utils.PropertiesUtil; import com.cloud.utils.server.ServerProperties; +import org.apache.cloudstack.context.TraceContextMdcWrapper; + /*** * The ServerDaemon class implements the embedded server, it can be started either * using JSVC or directly from the JAR along with additional jars not shaded in the uber-jar. @@ -108,6 +110,9 @@ public class ServerDaemon implements Daemon { ////////////////////////////////////////////////// public static void main(final String... anArgs) throws Exception { + // Install the trace-context to MDC hook before the server starts, so every + // thread with an active OpenTelemetry span carries mosaic_trace_id in its logs. + TraceContextMdcWrapper.register(); final ServerDaemon daemon = new ServerDaemon(); daemon.init(null); daemon.start(); diff --git a/client/src/main/webapp/WEB-INF/web.xml b/client/src/main/webapp/WEB-INF/web.xml index 43bee7e59d88..fdb899b55562 100644 --- a/client/src/main/webapp/WEB-INF/web.xml +++ b/client/src/main/webapp/WEB-INF/web.xml @@ -36,6 +36,16 @@ classpath:META-INF/cloudstack/webApplicationContext.xml + + apiTraceFilter + org.apache.cloudstack.api.filter.ApiTraceFilter + + + + apiTraceFilter + /api/* + + cloudStartupServlet com.cloud.servlet.CloudStartupServlet