From 44e4619eaf72430c01ddb1f6360b61f4505fc22a Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Mon, 11 May 2026 14:42:15 +0300 Subject: [PATCH 1/4] Remove summaries from updated fields in patient CDC events --- .../clinic/templates/5-patients-source-kafka-connector.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml b/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml index 6857b9b76..98cf3ea3f 100644 --- a/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml +++ b/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml @@ -14,8 +14,8 @@ spec: collection: patients connection.uri: {{ .Values.global.kafka.connect.mongoConnectionUri }} copy.existing: false - pipeline: '[ {$project: {"fullDocument.summary": 0}} ]' - startup.mode.copy.existing.pipeline: '[ {$project: {"fullDocument.summary": 0}} ]' + pipeline: '[ {$project: {"fullDocument.summary": 0, "updateDescription.updatedFields.summary": 0}} ]' + startup.mode.copy.existing.pipeline: '[ {$project: {"fullDocument.summary": 0, "updateDescription.updatedFields.summary": 0}} ]' database: clinic key.converter: org.apache.kafka.connect.json.JsonConverter key.converter.schemas.enable: false From fd5eec3441867b7730e52ad1546e111e5306a380 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 10 Jun 2026 11:50:34 +0300 Subject: [PATCH 2/4] Add user activity table --- .../templates/2-users-source-kafka-connector.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index 85717257c..33015bcff 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -27,11 +27,17 @@ spec: value.converter: 'org.apache.kafka.connect.json.JsonConverter' value.converter.schemas.enable: false snapshot.mode: {{ .Values.keycloak.snapshotMode }} - table.include.list: 'public.user_entity,public.user_role_mapping,public.keycloak_role,public.user_attribute' - transforms: 'filter,filter_user_attr' + table.include.list: 'public.user_entity,public.user_role_mapping,public.keycloak_role,public.user_attribute,public.tidepool_user_activity_event' + # Key the user-activity outbox by user_id (rather than the row's PK) so every + # event for a user lands on one partition and is consumed in commit order. This + # is what lets the clinic-worker consumer apply updates as last-writer-wins + # safely; combined with snapshot.mode=never it removes out-of-order/replay + # regressions. Other tables keep their default primary-key based message key. + message.key.columns: 'public.tidepool_user_activity_event:user_id' + transforms: 'filter' transforms.filter.type: 'io.debezium.transforms.Filter' transforms.filter.language: 'jsr223.groovy' - transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role)' + transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role|tidepool_user_activity_event)' transforms.filter.condition: "value.op && (((value.op == 'r' || value.op == 'c' || value.op == 'u') && (value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}')) || (value.op == 'd'))" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy' From 5ed0ff2a2588f651788c2f6ed3a2b91572e355b4 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 10 Jun 2026 12:03:17 +0300 Subject: [PATCH 3/4] Add user activity events --- .../templates/2-users-source-kafka-connector.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index 33015bcff..e889efc06 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -34,11 +34,20 @@ spec: # safely; combined with snapshot.mode=never it removes out-of-order/replay # regressions. Other tables keep their default primary-key based message key. message.key.columns: 'public.tidepool_user_activity_event:user_id' - transforms: 'filter' + transforms: 'filter,user_activity_filter,filter_user_attr' transforms.filter.type: 'io.debezium.transforms.Filter' transforms.filter.language: 'jsr223.groovy' - transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role|tidepool_user_activity_event)' + transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role)' transforms.filter.condition: "value.op && (((value.op == 'r' || value.op == 'c' || value.op == 'u') && (value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}')) || (value.op == 'd'))" + # Realm filter scoped to the user-activity outbox only. Keeps inserts (and any + # snapshot reads/updates) for the configured realm and drops everything else, + # including pruning deletes (their `after` is null, so they fail the check) — + # the clinic-worker consumer ignores deletes anyway. Records on other topics do + # not match this regex and pass through untouched. + transforms.user_activity_filter.type: 'io.debezium.transforms.Filter' + transforms.user_activity_filter.language: 'jsr223.groovy' + transforms.user_activity_filter.topic.regex: '.+\.public.tidepool_user_activity_event' + transforms.user_activity_filter.condition: "value.op && (value.op == 'r' || value.op == 'c' || value.op == 'u') && value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}'" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy' transforms.filter_user_attr.topic.regex: '.+\.public.(user_attribute)' From d9adfc500435381b6f895cee6db331a56a2e860a Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 11 Jun 2026 17:22:05 +0300 Subject: [PATCH 4/4] Drop user-activity tombstones in connector filter Set null.handling.mode=drop on user_activity_filter so pruning-delete tombstones are discarded at the source instead of reaching the consumer. Scoped to this filter's topic.regex, so keycloak tables are unaffected. --- .../kafka/templates/2-users-source-kafka-connector.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index e889efc06..016741ec9 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -47,6 +47,10 @@ spec: transforms.user_activity_filter.type: 'io.debezium.transforms.Filter' transforms.user_activity_filter.language: 'jsr223.groovy' transforms.user_activity_filter.topic.regex: '.+\.public.tidepool_user_activity_event' + # Drop pruning-delete tombstones (null value) for this topic instead of the + # default 'keep', so they don't reach the consumer. Scoped to this filter's + # topic.regex, so the keycloak tables' tombstones are unaffected. + transforms.user_activity_filter.null.handling.mode: 'drop' transforms.user_activity_filter.condition: "value.op && (value.op == 'r' || value.op == 'c' || value.op == 'u') && value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}'" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy'