Add new extension yagp hooks collector

.github/workflows/build-cloudberry.yml

@@ -271,6 +271,10 @@ jobs:
                 },
                 "enable_core_check":false
               },
+              {"test":"gpcontrib-yagp-hooks-collector",
+               "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"],
+               "extension":"yagp_hooks_collector"
+              },
               {"test":"ic-expandshrink",
                "make_configs":["src/test/isolation2:installcheck-expandshrink"]
               },
@@ -535,10 +539,11 @@ jobs:
         if: needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector
         run: |
           set -eo pipefail
           chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
-          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
             echo "::error::Configure script failed"
             exit 1
           fi
@@ -1403,6 +1408,7 @@ jobs:
         if: success() && needs.check-skip.outputs.should_skip != 'true'
         env:
           SRC_DIR: ${{ github.workspace }}
+          BUILD_DESTINATION: /usr/local/cloudberry-db
         shell: bash {0}
         run: |
           set -o pipefail
@@ -1432,6 +1438,30 @@ jobs:
             PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}"
           fi
 
+          # Create extension if required
+          if [[ "${{ matrix.extension != '' }}" == "true" ]]; then
+            case "${{ matrix.extension }}" in
+              yagp_hooks_collector)
+                if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \
+                  source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \
+                  gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \
+                  gpstop -ra && \
+                  echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \
+                        SHOW shared_preload_libraries; \
+                        TABLE pg_extension;' | \
+                    psql postgres"
+                then
+                    echo "Error creating yagp_hooks_collector extension"
+                    exit 1
+                fi
+                ;;
+              *)
+                echo "Unknown extension: ${{ matrix.extension }}"
+                exit 1
+                ;;
+            esac
+          fi
+
           if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then
             PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}"
           fi

.gitignore

@@ -73,4 +73,9 @@ lib*.pc
 /compile_commands.json
 /tmp_install/
 /.cache/
-/install/
+/install/
+*.o
+*.so
+src/protos/
+.vscode
+compile_commands.json

Makefile

@@ -3,14 +3,12 @@
 # to build Postgres with a different make, we have this make file
 # that, as a service, will look for a GNU make and invoke it, or show
 # an error message if none could be found.
-
 # If the user were using GNU make now, this file would not get used
 # because GNU make uses a make file named "GNUmakefile" in preference
 # to "Makefile" if it exists. PostgreSQL is shipped with a
 # "GNUmakefile". If the user hasn't run the configure script yet, the
 # GNUmakefile won't exist yet, so we catch that case as well.
 
-
 # AIX make defaults to building *every* target of the first rule.  Start with
 # a single-target, empty rule to make the other targets non-default.
 all:

configure

@@ -722,6 +722,7 @@ with_apr_config
 with_libcurl
 with_rt
 with_zstd
+with_yagp_hooks_collector
 with_libbz2
 LZ4_LIBS
 LZ4_CFLAGS
@@ -942,6 +943,7 @@ with_zlib
 with_lz4
 with_libbz2
 with_zstd
+with_yagp_hooks_collector
 with_rt
 with_libcurl
 with_apr_config
@@ -11150,6 +11152,32 @@ $as_echo "yes" >&6; }
 fi
 fi
 
+#
+# yagp_hooks_collector
+#
+
+
+
+# Check whether --with-yagp-hooks-collector was given.
+if test "${with_yagp_hooks_collector+set}" = set; then :
+  withval=$with_yagp_hooks_collector;
+  case $withval in
+    yes)
+      :
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-yagp-hooks-collector option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_yagp_hooks_collector=no
+
+fi
+
 #
 # Realtime library
 #

configure.ac

@@ -1365,6 +1365,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard],
 AC_MSG_RESULT([$with_zstd])
 AC_SUBST(with_zstd)
 
+#
+# yagp_hooks_collector
+#
+PGAC_ARG_BOOL(with, yagp_hooks_collector, no,
+              [build with YAGP hooks collector extension])
+AC_SUBST(with_yagp_hooks_collector)
+
 if test "$with_zstd" = yes; then
   dnl zstd_errors.h was renamed from error_public.h in v1.4.0
   PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0])

devops/build/automation/cloudberry/scripts/configure-cloudberry.sh

@@ -53,6 +53,7 @@
 #
 # Optional Environment Variables:
 #   LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
+#   CONFIGURE_EXTRA_OPTS - Args to pass to configure command
 #   ENABLE_DEBUG - Enable debug build options (true/false, defaults to
 #                  false)
 #
@@ -177,7 +178,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \
             --with-uuid=e2fs \
             ${CONFIGURE_MDBLOCALES_OPTS} \
             --with-includes=/usr/local/xerces-c/include \
-            --with-libraries=${BUILD_DESTINATION}/lib || exit 4
+            --with-libraries=${BUILD_DESTINATION}/lib \
+            ${CONFIGURE_EXTRA_OPTS:-""} || exit 4
 log_section_end "Configure"
 
 # Capture version information

gpcontrib/Makefile

@@ -35,6 +35,9 @@ else
                diskquota
 endif
 
+ifeq "$(with_yagp_hooks_collector)" "yes"
+	recurse_targets += yagp_hooks_collector
+endif
 ifeq "$(with_zstd)" "yes"
 	recurse_targets += zstd
 endif

gpcontrib/yagp_hooks_collector/.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: LLVM
+SortIncludes: false

gpcontrib/yagp_hooks_collector/Makefile

@@ -0,0 +1,41 @@
+MODULE_big = yagp_hooks_collector
+EXTENSION  = yagp_hooks_collector
+DATA       = $(wildcard *--*.sql)
+REGRESS    = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds yagp_locale
+
+PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service
+PROTO_OBJS  = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES))
+
+C_OBJS   = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c))
+CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp))
+OBJS     = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS)
+
+override CXXFLAGS = -Werror -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \
+	-Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \
+	-Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \
+	-Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD
+
+PG_CXXFLAGS += -Isrc -Iinclude
+SHLIB_LINK += -lprotobuf -lpthread -lstdc++
+EXTRA_CLEAN = src/protos
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = gpcontrib/yagp_hooks_collector
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto
+	@mkdir -p src/protos
+	sed -i 's/optional //g' $^
+	sed -i 's|cloud/mdb/yagpcc/api/proto/common/|protos/|g' $^
+	protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^
+	mv src/protos/$*.pb.cc src/protos/$*.pb.cpp
+
+$(CPP_OBJS): src/protos/yagpcc_metrics.pb.h src/protos/yagpcc_plan.pb.h src/protos/yagpcc_set_service.pb.h
+src/protos/yagpcc_set_service.pb.o: src/protos/yagpcc_metrics.pb.h

gpcontrib/yagp_hooks_collector/README.md

@@ -0,0 +1,28 @@
+## YAGP Hooks Collector
+
+An extension for collecting greenplum query execution metrics and reporting them to an external agent.
+
+### Collected Statistics
+
+#### 1. Query Lifecycle
+-   **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info.
+-   **GUC:** `yagpcc.enable`.
+
+#### 2. `EXPLAIN` data
+-   **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it.
+-   **GUC:** `yagpcc.enable`.
+
+#### 3. `EXPLAIN ANALYZE` data
+-   **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it.
+-   **GUCs:** `yagpcc.enable`, `yagpcc.min_analyze_time`, `yagpcc.enable_cdbstats`(ANALYZE), `yagpcc.enable_analyze`(BUFFERS, TIMING, VERBOSE).
+
+#### 4. Other Metrics
+-   **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics.
+-   **GUC:** `yagpcc.enable`.
+
+### General Configuration
+-   **Nested Queries:** When `yagpcc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates.
+-   **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `yagpcc.uds_path`.
+-   **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `yagpcc.ignored_users_list`.
+-   **Trimming plans:** Query texts and execution plans are trimmed based on `yagpcc.max_text_size` and `yagpcc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB.
+-   **Analyze collection:** Analyze is sent if execution time exceeds `yagpcc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `yagpcc.enable_analyze` is true.