fix(webapp): dashboard agent maintenance moves into the agent project

.server-changes/dashboard-agent-maintenance-move.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Routine cleanup of old dashboard agent data now runs on its own schedule.

apps/webapp/app/services/dashboardAgentChatRetention.server.ts

@@ -1,71 +1,14 @@
 /**
- * Retention for soft-deleted chats. A deleted chat is kept for a grace window and then
- * hard-deleted with all its child rows; one bounded statement per run, oldest first.
- * Also the eventual purge behind organization deletion, which soft-deletes the org's
- * chats so this same sweep removes them.
+ * The purge behind organization deletion: it soft-deletes the org's chats, and retention
+ * hard-deletes them once the window passes.
  */
 
-import {
-  hardDeleteChatsSoftDeletedBefore,
-  softDeleteChatsForOrganization,
-} from "@internal/dashboard-agent-db";
+import { softDeleteChatsForOrganization } from "@internal/dashboard-agent-db";
 import { dashboardAgentDb } from "~/services/dashboardAgentDb.server";
-import { logger } from "~/services/logger.server";
 
 /**
- * How long a soft-deleted chat is kept before it and its children are hard-deleted.
- * Long enough that an accidental delete can still be investigated; org deletion soft-
- * deletes the org's chats, so those are removed the same way once the window passes.
- */
-export const CHAT_SOFT_DELETE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
-
-/** Per-run cap. Retention is one bounded statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
-export type ChatRetentionResult = {
-  /** Soft-deleted chats past the retention window dropped this run. */
-  purged: number;
-  failed: number;
-};
-
-export type ChatRetentionDeps = {
-  now?: () => Date;
-  limit?: number;
-  /** Hard-delete chats soft-deleted before `before`. Returns how many went. */
-  purge?: (params: { before: Date; limit: number }) => Promise<number>;
-};
-
-export async function sweepDashboardAgentSoftDeletedChats(
-  deps: ChatRetentionDeps = {}
-): Promise<ChatRetentionResult> {
-  const now = deps.now?.() ?? new Date();
-  const limit = deps.limit ?? RETENTION_BATCH_LIMIT;
-  const purge =
-    deps.purge ?? ((params) => hardDeleteChatsSoftDeletedBefore(dashboardAgentDb, params));
-
-  const result: ChatRetentionResult = { purged: 0, failed: 0 };
-
-  try {
-    result.purged = await purge({
-      before: new Date(now.getTime() - CHAT_SOFT_DELETE_RETENTION_MS),
-      limit,
-    });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent chat retention failed", { error });
-  }
-
-  if (result.failed > 0) {
-    throw new Error("The dashboard agent chat retention pass failed");
-  }
-
-  return result;
-}
-
-/**
- * Soft-delete every chat belonging to a deleted organization. The retention sweep above
- * hard-deletes them once the window passes, so the org-deletion request never runs a
- * cross-database hard delete.
+ * Soft-delete every chat belonging to a deleted organization. Retention hard-deletes them
+ * once the window passes, so the org-deletion request never runs a cross-database hard delete.
  */
 export async function purgeDashboardAgentChatsForOrganization(params: {
   organizationId: string;

apps/webapp/app/services/dashboardAgentWatchSweep.server.ts

@@ -6,8 +6,6 @@
 import {
   cancelWatch,
   claimWatchAlertDispatch,
-  deleteTerminalWatchesOlderThan,
-  deleteWatchSubmissionsOlderThan,
   listExpiredActiveWatches,
   listWatchBatchGroupsToArm,
   listWatchesAwaitingDelivery,
@@ -57,12 +55,6 @@ export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000;
 /** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */
 const SWEEP_BATCH_LIMIT = 100;
 
-/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */
-export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
-
-/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
 /**
  * How many rows one sweep handles at once. An incident expires a whole group together, and a
  * bound is what stops one slow tenant spending the entire visibility window.
@@ -91,10 +83,6 @@ export type WatchSweepResult = {
   redelivered: number;
   /** Decided but not handed over, with no agent project. They stay owed. */
   deliveryDeferred: number;
-  /** Long-terminal rows dropped by retention. */
-  purged: number;
-  /** Ledger rows dropped by retention. */
-  purgedSubmissions: number;
   failed: number;
 };
 
@@ -113,10 +101,6 @@ export type WatchSweepDeps = {
   deliver?: (watch: Watch) => Promise<void>;
   /** Gates the delivery half only. Finalization never depends on it. */
   configured?: () => boolean;
-  /** Drop terminal rows older than `before`. Returns how many went. */
-  purgeTerminal?: (params: { before: Date; limit: number }) => Promise<number>;
-  /** Drop submission-ledger rows older than `before`. */
-  purgeSubmissions?: (params: { before: Date; limit: number }) => Promise<number>;
   /** How many rows are handled at once. */
   concurrency?: number;
 };
@@ -332,11 +316,6 @@ export async function sweepDashboardAgentWatches(
   const listAwaitingDelivery =
     deps.listAwaitingDelivery ??
     ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params));
-  const purgeTerminal =
-    deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params));
-  const purgeSubmissions =
-    deps.purgeSubmissions ??
-    ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params));
 
   const result: WatchSweepResult = {
     overdue: 0,
@@ -347,8 +326,6 @@ export async function sweepDashboardAgentWatches(
     undelivered: 0,
     redelivered: 0,
     deliveryDeferred: 0,
-    purged: 0,
-    purgedSubmissions: 0,
     failed: 0,
   };
 
@@ -435,18 +412,6 @@ export async function sweepDashboardAgentWatches(
     }
   }
 
-  // Retention runs last, over rows both halves are finished with. Its own try/catch so a
-  // lost retention pass can't mask the other failures.
-  try {
-    const before = new Date(now.getTime() - WATCH_RETENTION_MS);
-    result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT });
-    // The ledger's rows age out on the same window: past it no client is still retrying.
-    result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error });
-  }
-
   if (result.failed > 0) {
     throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`);
   }

apps/webapp/app/v3/commonWorker.server.ts

@@ -11,12 +11,7 @@ import {
   runAttioUserSync,
   runAttioWorkspaceSync,
 } from "~/services/attio.server";
-import {
-  purgeDashboardAgentChatsForOrganization,
-  sweepDashboardAgentSoftDeletedChats,
-} from "~/services/dashboardAgentChatRetention.server";
-import { sweepDashboardAgentTurnEvals } from "~/services/dashboardAgentEvalRetention.server";
-import { sweepDashboardAgentInvestigations } from "~/services/dashboardAgentInvestigationSweep.server";
+import { purgeDashboardAgentChatsForOrganization } from "~/services/dashboardAgentChatRetention.server";
 import {
   rearmDashboardAgentWatchBatches,
   sweepDashboardAgentWatches,
@@ -47,7 +42,7 @@ function initializeWorker() {
 
   logger.debug(`👨‍🏭 Initializing common worker at host ${env.COMMON_WORKER_REDIS_HOST}`);
 
-  // Only schedule the agent maintenance cron where the agent is actually set up. Otherwise
+  // Only schedule the agent watch cron where the agent is actually set up. Otherwise
   // its sweeps hit a missing schema and drip a dead-letter entry every run.
   const dashboardAgentConfigured =
     env.DASHBOARD_AGENT_ENABLED === "1" || Boolean(env.DASHBOARD_AGENT_DATABASE_URL);
@@ -161,16 +156,15 @@ function initializeWorker() {
           maxAttempts: 5,
         },
       },
-      // Stuck investigation cards and turn-eval retention.
+      // @deprecated, moved to the dashboard agent project; remove once the queue drains.
       "dashboardAgent.maintenance": {
         schema: CronSchema,
-        visibilityTimeoutMs: 60_000 * 5,
-        ...(dashboardAgentConfigured ? { cron: "*/5 * * * *", jitterInMs: 30_000 } : {}),
+        visibilityTimeoutMs: 60_000,
         retry: {
           maxAttempts: 1,
         },
       },
-      // The watch backstops: expiry, wake redelivery, retention and dead batch chains.
+      // The watch backstops: expiry, wake redelivery and dead batch chains.
       "dashboardAgent.watchMaintenance": {
         schema: CronSchema,
         visibilityTimeoutMs: 60_000 * 5,
@@ -179,7 +173,7 @@ function initializeWorker() {
           maxAttempts: 1,
         },
       },
-      // Soft-deletes a deleted organization's chats; the maintenance sweep purges them.
+      // Soft-deletes a deleted organization's chats; retention hard-deletes them later.
       "dashboardAgent.purgeOrganization": {
         schema: z.object({
           organizationId: z.string(),
@@ -247,48 +241,15 @@ function initializeWorker() {
         const service = new BulkActionService();
         await service.process(payload.bulkActionId);
       },
-      "dashboardAgent.maintenance": async () => {
-        // Each backstop runs independently; the first failure is rethrown at the end.
-        let failure: unknown;
-
-        try {
-          const investigations = await sweepDashboardAgentInvestigations();
-          if (investigations.stale > 0) {
-            logger.debug("Dashboard agent investigation sweep", investigations);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        // Retention on the judged-turn rows. Independent of the agent being configured.
-        try {
-          const evals = await sweepDashboardAgentTurnEvals();
-          if (evals.purged > 0) {
-            logger.debug("Dashboard agent turn-eval retention", evals);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        // Hard-delete chats soft-deleted past the retention window, with their children.
-        try {
-          const chats = await sweepDashboardAgentSoftDeletedChats();
-          if (chats.purged > 0) {
-            logger.debug("Dashboard agent chat retention", chats);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        if (failure) throw failure;
-      },
+      // @deprecated, moved to the dashboard agent project; remove once the queue drains.
+      "dashboardAgent.maintenance": async () => {},
       "dashboardAgent.watchMaintenance": async () => {
         // Each backstop runs independently; the first failure is rethrown at the end.
         let failure: unknown;
 
         try {
           const watches = await sweepDashboardAgentWatches();
-          if (watches.overdue > 0 || watches.undelivered > 0 || watches.purged > 0) {
+          if (watches.overdue > 0 || watches.undelivered > 0) {
             logger.debug("Dashboard agent watch sweep", watches);
           }
         } catch (error) {