perf(clickhouse): stop storing native event attributes

Author: carderne

internal-packages/clickhouse/schema/042_reduce_task_events_v2_storage_overhead.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+
+-- Full-text search is served outside the source event table. Keeping these
+-- indexes here adds work to every event insert and merge without serving reads.
+ALTER TABLE trigger_dev.task_events_v2
+    DROP INDEX IF EXISTS idx_attributes_text_search;
+
+ALTER TABLE trigger_dev.task_events_v2
+    DROP INDEX IF EXISTS idx_message_text_search;
+
+-- attributes remains an insert input for attributes_text, but is no longer
+-- stored. Writers must include attributes in an explicit insert column list
+-- because implicit INSERT column lists exclude EPHEMERAL columns.
+ALTER TABLE trigger_dev.task_events_v2
+    MODIFY COLUMN attributes JSON EPHEMERAL;
+
+-- +goose Down
+
+-- Restoring the stored JSON column safely requires inspecting the live schema
+-- and coordinating the writer rollback. Use a new forward migration instead.
+SELECT throwIf(1, 'This migration cannot be rolled back automatically');

internal-packages/clickhouse/src/taskEvents.test.ts

@@ -62,6 +62,49 @@ describe("task events v2", () => {
           has_inserted_at: 1,
         },
       ]);
+
+      const readColumnKinds = ch.reader.query({
+        name: "read-task-event-attribute-column-kinds",
+        query: `SELECT name, default_kind, default_expression
+        FROM system.columns
+        WHERE database = 'trigger_dev'
+          AND table = 'task_events_v2'
+          AND name IN ('attributes', 'attributes_text')
+        ORDER BY name`,
+        schema: z.object({
+          name: z.string(),
+          default_kind: z.string(),
+          default_expression: z.string(),
+        }),
+      });
+      const [columnError, columns] = await readColumnKinds({});
+      expect(columnError).toBeNull();
+      expect(columns).toEqual([
+        {
+          name: "attributes",
+          default_kind: "EPHEMERAL",
+          default_expression: "defaultValueOfTypeName('JSON')",
+        },
+        {
+          name: "attributes_text",
+          default_kind: "MATERIALIZED",
+          default_expression: "toJSONString(attributes)",
+        },
+      ]);
+
+      const readRemovedIndexes = ch.reader.query({
+        name: "read-removed-task-event-text-indexes",
+        query: `SELECT name
+        FROM system.data_skipping_indices
+        WHERE database = 'trigger_dev'
+          AND table = 'task_events_v2'
+          AND name IN ('idx_attributes_text_search', 'idx_message_text_search')
+        ORDER BY name`,
+        schema: z.object({ name: z.string() }),
+      });
+      const [indexError, indexes] = await readRemovedIndexes({});
+      expect(indexError).toBeNull();
+      expect(indexes).toEqual([]);
     }
   );
 });