fix(hosting): disable ClickHouse system-log telemetry tables and apply profile settings via users.d

.server-changes/hosting-clickhouse-system-logs.md

@@ -0,0 +1,6 @@
+---
+area: hosting
+type: fix
+---
+
+Self-hosted ClickHouse now disables its unbounded internal telemetry tables and keeps query/error logs on a bounded retention, fixing runaway CPU and memory usage on the recommended machine size. Existing self-hosted deployments must drop the previously written disabled log tables separately to reclaim disk.

hosting/docker/clickhouse/override.xml

@@ -9,12 +9,42 @@
     <!-- Official recommendations for systems with <16GB RAM -->
     <mark_cache_size>524288000</mark_cache_size> <!-- 500MB -->
     <concurrent_threads_soft_limit_num>1</concurrent_threads_soft_limit_num>
-    <profiles>
-        <default>
-        <max_block_size>8192</max_block_size>
-        <max_download_threads>1</max_download_threads>
-        <input_format_parallel_parsing>0</input_format_parallel_parsing>
-        <output_format_parallel_formatting>0</output_format_parallel_formatting>
-        </default>
-    </profiles>
-</clickhouse>
+
+    <!-- ClickHouse's own telemetry tables have no TTL and grow without bound; on the
+         recommended webapp machine size their background merges eventually stop fitting
+         in memory and are retried forever (there is no backoff for failed merges),
+         pinning the CPU and ultimately failing the webapp's own inserts (#4343).
+         The ClickHouse low-RAM guide recommends disabling exactly these tables:
+         https://clickhouse.com/docs/operations/tips
+         Same list as the dev stack (docker/config/clickhouse-disable-system-logs.xml),
+         extended with the newer log tables. -->
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <trace_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <session_log remove="1"/>
+    <text_log remove="1"/>
+    <zookeeper_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <asynchronous_insert_log remove="1"/>
+    <backup_log remove="1"/>
+    <latency_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <query_views_log remove="1"/>
+
+    <!-- Keep query_log and error_log (small and useful for debugging), but bounded.
+         A config-level TTL survives log-table recreation, unlike ALTER ... MODIFY TTL.
+         Note: ClickHouse renames the existing table to *_log_<N> when this changes;
+         drop those leftovers to reclaim disk. -->
+    <query_log>
+        <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
+    </query_log>
+    <error_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </error_log>
+
+    <!-- Profile settings do NOT live here: config.d is silently ignored for them.
+         See users-override.xml, mounted under users.d. -->
+</clickhouse>

hosting/docker/clickhouse/users-override.xml

@@ -0,0 +1,21 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <!-- Low-memory settings for the recommended webapp machine size. These were
+                 previously in config.d/override.xml where profile settings are silently
+                 ignored — they only take effect from the users config tree (#4343). -->
+            <max_block_size>8192</max_block_size>
+            <max_download_threads>1</max_download_threads>
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+            <!-- The memory/query profilers sample stacks into system.trace_log
+                 (a Memory/MemoryPeak sample every 4 MB allocated, by default) — the main
+                 firehose that fills it. trace_log is disabled in override.xml; turn the
+                 samplers off too so they don't burn CPU producing discarded rows. -->
+            <memory_profiler_step>0</memory_profiler_step>
+            <memory_profiler_sample_probability>0</memory_profiler_sample_probability>
+            <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>
+            <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
+        </default>
+    </profiles>
+</clickhouse>

hosting/docker/webapp/docker-compose.yml

@@ -177,6 +177,7 @@ services:
       - clickhouse:/var/lib/clickhouse
       - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro
       - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro
+      - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro
     networks:
       - webapp
     healthcheck: