From 7996890ac4976989a3674d34ebc539c5678431e1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 01:00:00 +0300 Subject: [PATCH 01/16] darwin: os_ops tests are updated --- tests/helpers/run_conditions.py | 10 ++++++ tests/test_os_ops_common.py | 60 +++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py index f847d879..ea20e5ba 100644 --- a/tests/helpers/run_conditions.py +++ b/tests/helpers/run_conditions.py @@ -11,3 +11,13 @@ class RunConditions: def skip_if_windows(): if platform.system().lower() == "windows": pytest.skip("This test does not support Windows.") + + @staticmethod + def skip_if_darwin(): + if platform.system().lower() == "darwin": + pytest.skip("This test does not support Darwin.") + + @staticmethod + def skip_if_linux(): + if platform.system().lower() == "linux": + pytest.skip("This test does not support Linux.") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 67f94146..45c39c6a 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -225,7 +225,7 @@ def test_get_platform__is_known(self, os_ops_descr: OsOpsDescr): p = os_ops.get_platform() assert p is not None assert type(p) is str - assert p in {"win32", "linux"} + assert p in {"win32", "linux", "darwin"} return def test_create_clone( @@ -422,11 +422,12 @@ def test_exec_command_with_exec_env__2(self, os_ops_descr: OsOpsDescr): assert not os_ops.path_exists(tmp_file) return - def test_exec_command_with_cwd(self, os_ops_descr: OsOpsDescr): + def test_exec_command_with_cwd__linux(self, os_ops_descr: OsOpsDescr): assert type(os_ops_descr) is OsOpsDescr assert isinstance(os_ops_descr.os_ops, OsOperations) RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() os_ops = os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) @@ -616,7 +617,7 @@ def test_makedirs_and_rmdirs_success( assert not os_ops.path_exists(path) return - def test_makedirs_failure( + def test_makedirs_failure__linux( self, os_ops_descr: OsOpsDescr, name_with_surprize: tagNameWithSurprize, @@ -633,6 +634,7 @@ def test_makedirs_failure( assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() path = "/root/test_dir-{}-{}".format( name_with_surprize.value, @@ -651,6 +653,42 @@ def test_makedirs_failure( __class__.helper__bug_check__unknown_os_ops_type(os_ops) return + def test_makedirs_failure__darwin( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test makedirs for failure. + """ + # Try to create a directory in a read-only location + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + RunConditions.skip_if_linux() + + path = "/root/test_dir-{}-{}".format( + name_with_surprize.value, + uuid.uuid4().bytes.hex(), + ) + + # Test makedirs + with pytest.raises(Exception) as x: + os_ops.makedirs(path) + + if type(os_ops).__name__ == "LocalOperations": + assert type(x.value) is OSError + elif type(os_ops).__name__ == "RemoteOperations": + assert type(x.value) is ExecUtilException + else: + __class__.helper__bug_check__unknown_os_ops_type(os_ops) + return + def test_listdir( self, os_ops_descr: OsOpsDescr, @@ -2559,13 +2597,16 @@ def test_is_abs_path__no( return # -------------------------------------------------------------------- - def test_get_abs_path( + def test_get_abs_path__linux( self, os_ops_descr: OsOpsDescr, ): assert type(os_ops_descr) is OsOpsDescr assert isinstance(os_ops_descr.os_ops, OsOperations) + RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() + os_ops = os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) @@ -3154,7 +3195,7 @@ def test_readlines__BIN( assert lines == result_bin return - def test_prove_environment_isolation( + def test_prove_environment_isolation__linux( self, os_ops_descr: OsOpsDescr, ): @@ -3166,6 +3207,9 @@ def test_prove_environment_isolation( os_ops = os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) + RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() + logging.info("=================== COKANUM PROOF START ===================") logging.info(f"Target environment type: [{os_ops_descr.sign}]") @@ -5130,7 +5174,7 @@ def test_popen_replace_env_of_os_ops( return - def test_popen_cwd( + def test_popen_cwd__linux( self, os_ops_descr: OsOpsDescr, ): @@ -5138,6 +5182,7 @@ def test_popen_cwd( assert isinstance(os_ops_descr.os_ops, OsOperations) RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() os_ops = os_ops_descr.os_ops @@ -5908,7 +5953,7 @@ def test_run_input( return - def test_run_cwd( + def test_run_cwd__linux( self, os_ops_descr: OsOpsDescr, ): @@ -5916,6 +5961,7 @@ def test_run_cwd( assert isinstance(os_ops_descr.os_ops, OsOperations) RunConditions.skip_if_windows() + RunConditions.skip_if_darwin() os_ops = os_ops_descr.os_ops From 44c9f82d3bf2fc58358034decd37c10376ab1a69 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 01:28:36 +0300 Subject: [PATCH 02/16] linux: internal_platform_utils.py is corrected 1) ProcessIsZombi_soft_check is corrected (raise) 2) Bad assert in _find_postmaster__throw_error__bad_line_format --- src/impl/platforms/linux/internal_platform_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py index 1afcc0fd..f0193952 100644 --- a/src/impl/platforms/linux/internal_platform_utils.py +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -246,6 +246,7 @@ def _FindPostmaster( pid_to_ppid, ) + # -------------------------------------------------------------------- def ProcessIsZombi_soft_check( self, os_ops: OsOperations, @@ -282,6 +283,7 @@ def ProcessIsZombi_soft_check( # If the file disappeared right during reading, it means the process is completely erased if __class__._is_file_not_found_exception(e): result = False + raise return result @@ -316,7 +318,7 @@ def _find_postmaster__throw_error__bad_line_format( ) -> typing.NoReturn: assert type(lines) is list assert type(i_line) is int - assert type(hint) is int + assert type(hint) is str error_lines: typing.List[str] = [] error_lines.append( From 068454a8a082fcf29466aae5349531214874ed41 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 01:29:24 +0300 Subject: [PATCH 03/16] darwin/internal_platform_utils.py is added --- .../darwin/internal_platform_utils.py | 427 ++++++++++++++++++ .../internal_platform_utils_factory.py | 4 + 2 files changed, 431 insertions(+) create mode 100644 src/impl/platforms/darwin/internal_platform_utils.py diff --git a/src/impl/platforms/darwin/internal_platform_utils.py b/src/impl/platforms/darwin/internal_platform_utils.py new file mode 100644 index 00000000..63365dcd --- /dev/null +++ b/src/impl/platforms/darwin/internal_platform_utils.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +from .. import internal_platform_utils as base +from ... import internal_utils +from ....raise_error import RaiseError + +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import OsCommandResult +from testgres.operations.exceptions import ExecUtilException +from testgres.operations.types import T_OS_EXEC_ENV + +import re +import shlex +import typing +import time + + +class InternalPlatformUtils(base.InternalPlatformUtils): + C_MAX_FIND_POSTMASTER_ATTEMPTS = 5 + C_BASH_EXE = "/bin/bash" + + sm_exec_env: T_OS_EXEC_ENV = { + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + } + + # -------------------------------------------------------------------- + def FindPostmaster( + self, + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> InternalPlatformUtils.FindPostmasterResult: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(__class__.C_BASH_EXE) is str + assert type(__class__.sm_exec_env) is dict + assert len(__class__.C_BASH_EXE) > 0 + assert len(bin_dir) > 0 + assert len(data_dir) > 0 + + failures: typing.List[Exception] = [] + + postmaster_pid: typing.Optional[int] = None + + nAttempts = 0 + + while True: + nAttempts += 1 + + try: + postmaster_pid = __class__._FindPostmaster( + os_ops, + bin_dir, + data_dir, + ) + except Exception as e: + failures.append(e) + + log_msg = "FindPostmaster (bin_dir={!r}, data_dir={!r}) detects a problem. Exception {}:\n{}".format( + bin_dir, + data_dir, + type(e).__name__, + e, + ) + internal_utils.send_log_debug(log_msg) + + if nAttempts < __class__.C_MAX_FIND_POSTMASTER_ATTEMPTS: + time.sleep(0.05) + continue + + __class__._find_postmaster__throw_error__fail( + bin_dir=bin_dir, + data_dir=data_dir, + failures=failures, + ) + + break + + if postmaster_pid is None: + return InternalPlatformUtils.FindPostmasterResult.create_not_found() + + assert type(postmaster_pid) is int + + return InternalPlatformUtils.FindPostmasterResult.create_ok(postmaster_pid) + + # -------------------------------------------------------------------- + @staticmethod + def _FindPostmaster( + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> typing.Optional[int]: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(__class__.C_BASH_EXE) is str + assert type(__class__.sm_exec_env) is dict + assert len(__class__.C_BASH_EXE) > 0 + assert len(bin_dir) > 0 + assert len(data_dir) > 0 + + pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres")) + data_dir_e = re.escape(data_dir) + + assert type(pg_path_e) is str + assert type(data_dir_e) is str + + # The regular expression remains the same since the output structure pid, ppid, args is the same + regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" + + # Change for macOS: Instead of Linux flag-combo "-ewwo" + # we use the standard POSIX "-eo", which works on macOS without trimming argument strings + cmd = [ + __class__.C_BASH_EXE, + "-c", + "ps -eo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp), + ] + + exec_r = os_ops.run( + cmd=cmd, + check=False, + exec_env=__class__.sm_exec_env, + ) + + assert type(exec_r) is OsCommandResult + assert type(exec_r.returncode) is int + assert type(exec_r.stdout) is bytes + assert type(exec_r.stderr) is bytes + + if exec_r.returncode == 1: + return None + + output = exec_r.stdout.decode("utf-8") + error = exec_r.stderr.decode("utf-8") + + assert type(output) is str + assert type(error) is str + + if exec_r.returncode != 0: + errMsg = f"test command returned an unexpected exit code: {exec_r.returncode}" + raise ExecUtilException( + message=errMsg, + command=cmd, + exit_code=exec_r.returncode, + out=output, + error=error, + ) + + lines = output.splitlines() + assert type(lines) is list + + if len(lines) == 0: + # ACHTUNG! + raise RuntimeError("Command returns 0 error code without output.") + + # parse result lines + pid_to_ppid: __class__.T_PID_TO_PPID = {} + + for i_line in range(len(lines)): + assert type(lines[i_line]) is str + + parts = lines[i_line].split() + assert type(parts) is list + + if len(parts) < 2: + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "no usefull data", + ) + + if not parts[0].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad pid", + ) + + if not parts[1].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad ppid", + ) + + pid = int(parts[0]) + ppid = int(parts[1]) + + if pid not in pid_to_ppid: + pid_to_ppid[pid] = ppid + continue + + other_ppid = pid_to_ppid[pid] + assert type(other_ppid) is int + + if ppid == other_ppid: + log_msg = "FindPostmaster (data_dir={!r}) get pid ({}) with ppid ({}) more than one time.".format( + data_dir, + pid, + ppid, + ) + internal_utils.send_log_debug(log_msg) + continue + + # ACTUNG ppid is changed --> restart + __class__._find_postmaster__throw_error__ppid_is_changed( + pid, + ppid, + other_ppid, + lines, + ) + + assert len(pid_to_ppid) <= len(lines) + + true_postmasters = [ + pid for pid, ppid in pid_to_ppid.items() + if ppid not in pid_to_ppid + ] + + if len(true_postmasters) == 0: + __class__._find_postmaster__throw_error__cycle( + pid_to_ppid, + ) + + if len(true_postmasters) == 1: + true_pid = true_postmasters[0] + + if len(pid_to_ppid) > 1: + msg = "Many processes like a postmaster for data dir [{}] are found ({}).".format( + data_dir, + len(true_postmasters), + ) + + msg += " List (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " True postmaster PID is {}.".format(true_pid) + internal_utils.send_log_debug(msg) + + return true_pid + + assert len(true_postmasters) > 1 + + __class__._find_postmaster__throw_error__many_postmasters( + true_postmasters, + pid_to_ppid, + ) + + # -------------------------------------------------------------------- + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + + # Change for macOS: Instead of reading non-existent /proc/PID/stat, + # we make a native call to the system ps and request the status (state) of the process. + cmd = ["ps", "-p", str(pid), "-o", "state="] + + try: + exec_r = os_ops.run( + cmd=cmd, + check=False, + exec_env=__class__.sm_exec_env, + ) + + assert type(exec_r) is OsCommandResult + + # Если процесс не найден (уже завершился и стерт), ps вернет код 1 + if exec_r.returncode != 0: + return False + + proc_status = exec_r.stdout.decode("utf-8", errors="ignore").strip() + + if not proc_status: + return False + + # В BSD-системах статус зомби обозначается буквой 'Z' + return proc_status.startswith("Z") + + except Exception as e: + if __class__._is_file_not_found_exception(e): + return False + raise + + @staticmethod + def _is_file_not_found_exception(e: Exception) -> bool: + if isinstance(e, FileNotFoundError): + return True + + if isinstance(e, ExecUtilException): + if e.exit_code == 2: + return True + return False + + T_PID_TO_PPID = typing.Dict[int, int] + + @staticmethod + def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str: + result = "" + sep = "" + for pid, ppid in pid_to_ppid.items(): + result += sep + " {}->{}".format(ppid, pid) + sep = ", " + return result + + @staticmethod + def _find_postmaster__throw_error__bad_line_format( + lines: typing.List[str], + i_line: int, + hint: str, + ) -> typing.NoReturn: + assert type(lines) is list + assert type(i_line) is int + assert type(hint) is str + + error_lines: typing.List[str] = [] + error_lines.append( + "Line {} has bad format. Hint: {}.".format( + i_line + 1, + hint, + ), + ) + error_lines.append( + "Problem line is:" + ) + error_lines.append( + " " + repr(lines[i_line]), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__ppid_is_changed( + pid: int, + ppid: int, + other_ppid: int, + lines: typing.List[str], + ) -> typing.NoReturn: + assert type(pid) is int + assert type(ppid) is int + assert type(other_ppid) is int + assert type(lines) is list + + error_lines: typing.List[str] = [] + error_lines.append( + "Parent of process ({}) is changed from {} to {}.".format( + pid, + other_ppid, + ppid, + ), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__cycle( + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Cycle in processes postgres process tree. " + + msg += " List (ppid->pid): {},".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " List size is {}.".format( + len(pid_to_ppid), + ) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__many_postmasters( + postmaster_pids: typing.List[int], + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Many processes like a postmaster are found ({}): {}.".format( + len(postmaster_pids), + ", ".join(map(str, postmaster_pids)), + ) + + msg += " Trees (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " Total process count is {}.".format(len(pid_to_ppid)) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__fail( + bin_dir: str, + data_dir: str, + failures: typing.List[Exception], + ) -> typing.NoReturn: + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(failures) is list + + method_name = "InternalPlatformUtils::FindPostmaster(bin_dir={!r}, data_dir={!r})".format( + bin_dir, + data_dir, + ) + + RaiseError.function_did_multiple_attempts_without_stable_result( + method_name, + failures, + ) diff --git a/src/impl/platforms/internal_platform_utils_factory.py b/src/impl/platforms/internal_platform_utils_factory.py index 1098185e..762e3bc1 100644 --- a/src/impl/platforms/internal_platform_utils_factory.py +++ b/src/impl/platforms/internal_platform_utils_factory.py @@ -19,5 +19,9 @@ def create_internal_platform_utils( from .win32 import internal_platform_utils as x return x.InternalPlatformUtils() + if platform_name == "darwin": + from .darwin import internal_platform_utils as x + return x.InternalPlatformUtils() + # not implemented return InternalPlatformUtils() From 8b19cc307e9f1c47c67d5de195e9b369a542bb18 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 01:43:15 +0300 Subject: [PATCH 04/16] CI: test-macos-14 is added --- .github/workflows/ci.yml | 70 +++++++++++++++++++- run_tests-darwin.sh | 134 +++++++++++++++++++++++++++++++++++++++ run_tests.sh | 13 ++-- 3 files changed, 210 insertions(+), 7 deletions(-) create mode 100755 run_tests-darwin.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c9295c1..cbf6da42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,8 @@ jobs: run: | twine check dist/* - test: + test-linux-container: + if: false runs-on: ubuntu-latest needs: build-check strategy: @@ -288,3 +289,70 @@ jobs: with: name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} path: "${{ env.RUN_CFG__LOGS_DIR }}/" + + test-macos-14: + runs-on: macos-14 + needs: build-check + strategy: + fail-fast: false + matrix: + # Testing stable versions of Python on Apple Silicon (M1) architecture + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + # Primary target version of PostgreSQL for the native environment + postgres: ["17"] + + env: + BASE_SIGN: "macos-py${{ matrix.python }}-pg${{ matrix.postgres }}" + + steps: + - name: Prepare variables + run: | + echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV + echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "---------- [$GITHUB_ENV]" + cat $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v7 + + - name: Prepare logs folder on the host + run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}" + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + + - name: Install PostgreSQL ${{ matrix.postgres }} via Homebrew + run: | + brew update + # Install the specified version of the DBMS + brew install postgresql@${{ matrix.postgres }} + + # Add postgres binaries (pg_ctl, initdb, etc.) to GITHUB_PATH, + # so that they are globally available for testing (Homebrew does not automatically link older/specific versions) + echo "$(brew --prefix postgresql@${{ matrix.postgres }})/bin" >> $GITHUB_PATH + + - name: Run native tests (Local operations only) + run: | + set -eux + echo "HELLO FROM MACOS RUNNER" + echo "HOME DIR IS [$(realpath ~/)]" + echo "WORK DIR IS [$(pwd)]" + + postgres --version + + export TEST_CFG__LOG_DIR="${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}" + + export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))" + export PYTHON_BINARY="python3" + + bash run_tests-darwin.sh + + - name: Upload Logs + uses: actions/upload-artifact@v7 + if: always() # Важно! Сохраняем логи даже при падении тестов + with: + name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} + path: "${{ env.RUN_CFG__LOGS_DIR }}/" diff --git a/run_tests-darwin.sh b/run_tests-darwin.sh new file mode 100755 index 00000000..5f72981c --- /dev/null +++ b/run_tests-darwin.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +set -eux + +# Filter tests for local execution (without remote/ssh) +if [ -z ${TEST_FILTER+x} ]; then + export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))" +fi + +# There is no nproc on macOS, so we use sysctl +echo NPROC: $(sysctl -n hw.ncpu) + +# Check for the presence of pg_config +echo check that pg_config is in PATH +command -v pg_config + +# Setting up the Python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +${PYTHON_BINARY} -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install --upgrade pip setuptools wheel +pip install -r tests/requirements.txt + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + +pip install coverage + +exec_command() { + local cmd="$1" + local prefix="$2" + + eval "$prefix $cmd" +} + +show_fs_state__impl() { + local prefix="$1" + local host_label="$2" + + set +x + echo "------------- ${host_label} FS STATE" + set -x + # Change for macOS: use the cross-platform -P flag instead of -T + exec_command "df -P" "$prefix" +} + +check_leftover_ports__impl() { + local prefix="$1" + local host_label="$2" + local ports_dir="/tmp/testgres/ports" + + set +x + echo "------------- Checking ${host_label} ports lock directory" + set -x + + # Check command: will print FOUND if the directory exists and is not empty + local check_cmd="if [ -d '${ports_dir}' ] && [ \"\$(ls -A '${ports_dir}' 2>/dev/null)\" ]; then echo 'FOUND'; fi" + + # Temporarily disable bash's instant drop (set +e) to safely intercept the result + set +e + local result + result=$(exec_command "$check_cmd" "$prefix") + set -e + + set +x + if [ "$result" = "FOUND" ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "ERROR: Leftover ports detected in $ports_dir on $host_label machine!" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + # We display a list of frozen ports so that the culprits can be identified + exec_command "ls -la '$ports_dir'" "$prefix" + + # We hard-drop the entire control script + # sleep 3600 + exit 1 + else + echo "Clear. No leftover port locks." + fi + set -x +} + +fs_verification() { + show_fs_state__impl "" "LOCAL" + + check_leftover_ports__impl "" "LOCAL" +} + +# ---------------------------------------- PATH + +fs_verification + +# run tests (PATH) +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- PG_BIN + +fs_verification + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- PG_CONFIG + +fs_verification + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- pg8000 + +fs_verification + +# test pg8000 +pip uninstall -y psycopg2 +pip install pg8000 +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- finish + +fs_verification + +# ---------------------------------------- coverage + +coverage report + +pip uninstall -y coverage diff --git a/run_tests.sh b/run_tests.sh index 0f98b60d..fc27833d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -2,17 +2,18 @@ set -eux -if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +# Filter tests for local execution (without remote/ssh) +if [ -z ${TEST_FILTER+x} ]; then + export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))" fi echo NPROC: $(nproc) -# fail early +# Check for the presence of pg_config echo check that pg_config is in PATH command -v pg_config -# prepare python environment +# Setting up the Python environment VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH ${PYTHON_BINARY} -m venv "${VENV_PATH}" @@ -86,7 +87,7 @@ check_leftover_ports__impl() { # We display a list of frozen ports so that the culprits can be identified exec_command "ls -la '$ports_dir'" "$prefix" - + # We hard-drop the entire control script # sleep 3600 exit 1 @@ -104,7 +105,7 @@ fs_verification__impl() { fs_verification() { fs_verification__impl "" "LOCAL" - + if [ -n "$REMOTE_SSH_PREFIX" ]; then fs_verification__impl "$REMOTE_SSH_PREFIX" "REMOTE" fi From 593b8e914ed7ab497433f0deb80a9eef0623f760 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:22:13 +0300 Subject: [PATCH 05/16] run_tests-darwin: pytest --color=yes --- run_tests-darwin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests-darwin.sh b/run_tests-darwin.sh index 5f72981c..bb3a78aa 100755 --- a/run_tests-darwin.sh +++ b/run_tests-darwin.sh @@ -103,7 +103,7 @@ fs_verification # run tests (PG_BIN) PG_BIN=$(pg_config --bindir) \ -time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n 4 --color=yes -k "${TEST_FILTER}" # ---------------------------------------- PG_CONFIG From 710bbaceb3a7c23e1ea6478380517495da726434 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:25:40 +0300 Subject: [PATCH 06/16] run_tests-darwin: pytest -n auto --- run_tests-darwin.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run_tests-darwin.sh b/run_tests-darwin.sh index bb3a78aa..4ed4a2ca 100755 --- a/run_tests-darwin.sh +++ b/run_tests-darwin.sh @@ -95,7 +95,7 @@ fs_verification() { fs_verification # run tests (PATH) -time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" # ---------------------------------------- PG_BIN @@ -103,7 +103,7 @@ fs_verification # run tests (PG_BIN) PG_BIN=$(pg_config --bindir) \ -time coverage run -a -m pytest -l -vvv -n 4 --color=yes -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto --color=yes -k "${TEST_FILTER}" # ---------------------------------------- PG_CONFIG @@ -111,7 +111,7 @@ fs_verification # run tests (PG_CONFIG) PG_CONFIG=$(pg_config --bindir)/pg_config \ -time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" # ---------------------------------------- pg8000 @@ -121,7 +121,7 @@ fs_verification pip uninstall -y psycopg2 pip install pg8000 PG_CONFIG=$(pg_config --bindir)/pg_config \ -time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" # ---------------------------------------- finish From 0985b9e46251ec4dc36272a00b13007bea461d0f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:32:56 +0300 Subject: [PATCH 07/16] fix: NodeApp::_gettempdir is corrected --- src/node_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node_app.py b/src/node_app.py index a6dfd602..8d6e40c2 100644 --- a/src/node_app.py +++ b/src/node_app.py @@ -300,7 +300,7 @@ def _gettempdir(self) -> str: # # Paranoid checks # - if type(v) is str: + if type(v) is not str: __class__._raise_bugcheck("os_ops.get_tempdir returned a value with type {}.".format( type(v).__name__, )) From 9ee80f641c220bba1e3bdb4467b3574a1919427e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:36:54 +0300 Subject: [PATCH 08/16] run_tests-darwin: pytest --color=yes (v2) --- run_tests-darwin.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests-darwin.sh b/run_tests-darwin.sh index 4ed4a2ca..21b1b320 100755 --- a/run_tests-darwin.sh +++ b/run_tests-darwin.sh @@ -95,7 +95,7 @@ fs_verification() { fs_verification # run tests (PATH) -time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto --color=yes -k "${TEST_FILTER}" # ---------------------------------------- PG_BIN @@ -111,7 +111,7 @@ fs_verification # run tests (PG_CONFIG) PG_CONFIG=$(pg_config --bindir)/pg_config \ -time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto --color=yes -k "${TEST_FILTER}" # ---------------------------------------- pg8000 @@ -121,7 +121,7 @@ fs_verification pip uninstall -y psycopg2 pip install pg8000 PG_CONFIG=$(pg_config --bindir)/pg_config \ -time coverage run -a -m pytest -l -vvv -n auto -k "${TEST_FILTER}" +time coverage run -a -m pytest -l -vvv -n auto --color=yes -k "${TEST_FILTER}" # ---------------------------------------- finish From 7fc5ff0d325dd73934123fcbfcb9e5d3d3225f6c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:42:35 +0300 Subject: [PATCH 09/16] macos: TEST_FILTER="not remote" --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbf6da42..d3159938 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,7 +345,7 @@ jobs: export TEST_CFG__LOG_DIR="${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}" - export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))" + export TEST_FILTER="not remote" export PYTHON_BINARY="python3" bash run_tests-darwin.sh From 2bcdf47d10f52c9a2bdf94a6de493d60327d0f31 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 02:57:28 +0300 Subject: [PATCH 10/16] darwin: internal_platform_utils is updated (sync with linux) --- src/impl/platforms/darwin/internal_platform_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/impl/platforms/darwin/internal_platform_utils.py b/src/impl/platforms/darwin/internal_platform_utils.py index 63365dcd..2bc0dc1f 100644 --- a/src/impl/platforms/darwin/internal_platform_utils.py +++ b/src/impl/platforms/darwin/internal_platform_utils.py @@ -284,6 +284,7 @@ def ProcessIsZombi_soft_check( return proc_status.startswith("Z") except Exception as e: + # If the file disappeared right during reading, it means the process is completely erased if __class__._is_file_not_found_exception(e): return False raise @@ -296,6 +297,7 @@ def _is_file_not_found_exception(e: Exception) -> bool: if isinstance(e, ExecUtilException): if e.exit_code == 2: return True + return False T_PID_TO_PPID = typing.Dict[int, int] @@ -307,6 +309,7 @@ def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str: for pid, ppid in pid_to_ppid.items(): result += sep + " {}->{}".format(ppid, pid) sep = ", " + continue return result @staticmethod From bcf2d9faad164255ca496d5c41cf5b354a595388 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 03:06:46 +0300 Subject: [PATCH 11/16] CI: run of "test-linux-container" is restored --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3159938..a4d887d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,6 @@ jobs: twine check dist/* test-linux-container: - if: false runs-on: ubuntu-latest needs: build-check strategy: From e3ccaf0e54ca7e5b69ce2a660dd18092a0948ac9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 03:19:29 +0300 Subject: [PATCH 12/16] linux::InternalPlatformUtils::ProcessIsZombi_soft_check is restored --- src/impl/platforms/linux/internal_platform_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py index f0193952..29dbb508 100644 --- a/src/impl/platforms/linux/internal_platform_utils.py +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -283,7 +283,6 @@ def ProcessIsZombi_soft_check( # If the file disappeared right during reading, it means the process is completely erased if __class__._is_file_not_found_exception(e): result = False - raise return result From a43444a056bf0b4a0763cbdf22d2ae3b7bac4139 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 03:20:04 +0300 Subject: [PATCH 13/16] darwin::InternalPlatformUtils::ProcessIsZombi_soft_check is corrected --- src/impl/platforms/darwin/internal_platform_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/impl/platforms/darwin/internal_platform_utils.py b/src/impl/platforms/darwin/internal_platform_utils.py index 2bc0dc1f..ba7d807b 100644 --- a/src/impl/platforms/darwin/internal_platform_utils.py +++ b/src/impl/platforms/darwin/internal_platform_utils.py @@ -287,7 +287,8 @@ def ProcessIsZombi_soft_check( # If the file disappeared right during reading, it means the process is completely erased if __class__._is_file_not_found_exception(e): return False - raise + + return None @staticmethod def _is_file_not_found_exception(e: Exception) -> bool: From ae2be9d1a4622495b9776c54092c041c3a514db5 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 03:29:31 +0300 Subject: [PATCH 14/16] CI: English --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4d887d3..397f5cbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -351,7 +351,7 @@ jobs: - name: Upload Logs uses: actions/upload-artifact@v7 - if: always() # Важно! Сохраняем логи даже при падении тестов + if: always() # IT IS IMPORTANT! with: name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} path: "${{ env.RUN_CFG__LOGS_DIR }}/" From e690318c10db85482e6eaf867bc9324e8f9505d1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 13:32:50 +0300 Subject: [PATCH 15/16] darwin::internal_platform_utils is rewritten _FindPostmaster - uses "-ewwo" (as in linux version) ProcessIsZombi_soft_check - new exact code, "-wwo" is used --- .../darwin/internal_platform_utils.py | 93 ++++++++++++------- 1 file changed, 59 insertions(+), 34 deletions(-) diff --git a/src/impl/platforms/darwin/internal_platform_utils.py b/src/impl/platforms/darwin/internal_platform_utils.py index ba7d807b..ce8fdb4f 100644 --- a/src/impl/platforms/darwin/internal_platform_utils.py +++ b/src/impl/platforms/darwin/internal_platform_utils.py @@ -13,6 +13,7 @@ import shlex import typing import time +import subprocess class InternalPlatformUtils(base.InternalPlatformUtils): @@ -107,15 +108,12 @@ def _FindPostmaster( assert type(pg_path_e) is str assert type(data_dir_e) is str - # The regular expression remains the same since the output structure pid, ppid, args is the same regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" - # Change for macOS: Instead of Linux flag-combo "-ewwo" - # we use the standard POSIX "-eo", which works on macOS without trimming argument strings cmd = [ __class__.C_BASH_EXE, "-c", - "ps -eo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp), + "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp), ] exec_r = os_ops.run( @@ -260,46 +258,73 @@ def ProcessIsZombi_soft_check( # Change for macOS: Instead of reading non-existent /proc/PID/stat, # we make a native call to the system ps and request the status (state) of the process. - cmd = ["ps", "-p", str(pid), "-o", "state="] + cmd = ["ps", "-p", str(pid), "-wwo", "state="] - try: - exec_r = os_ops.run( - cmd=cmd, - check=False, - exec_env=__class__.sm_exec_env, - ) - - assert type(exec_r) is OsCommandResult - - # Если процесс не найден (уже завершился и стерт), ps вернет код 1 - if exec_r.returncode != 0: - return False + exec_r = os_ops.run( + cmd=cmd, + check=False, + exec_env=__class__.sm_exec_env, + stdout=subprocess.PIPE, + ) - proc_status = exec_r.stdout.decode("utf-8", errors="ignore").strip() + assert type(exec_r) is OsCommandResult + assert type(exec_r.stdout) is bytes - if not proc_status: - return False + # If the process is not found (already terminated and deleted), ps will return code 1 + if exec_r.returncode == 1: + return False - # В BSD-системах статус зомби обозначается буквой 'Z' - return proc_status.startswith("Z") + if exec_r.returncode != 0: + return None - except Exception as e: - # If the file disappeared right during reading, it means the process is completely erased - if __class__._is_file_not_found_exception(e): - return False + proc_status = exec_r.stdout.decode("utf-8").rstrip() - return None + if not proc_status: + return None - @staticmethod - def _is_file_not_found_exception(e: Exception) -> bool: - if isinstance(e, FileNotFoundError): + assert len(proc_status) > 0 + + # state The state is given by a sequence of letters, for example, + # "RWNA". The first letter indicates the run state of the process: + # + # D Marks a process in disk (or other short term, uninterruptible) wait.[legacy option] + # I Marks a process that is idle (sleeping for longer than about 20 seconds). + # R Marks a runnable process. + # S Marks a process that is sleeping for less than about 20 seconds. + # T Marks a stopped process. + # U Marks a process in uninterruptible wait. + # Z Marks a dead process (a 'zombie'). + # + # Additional characters after these, if any, indicate additional + # state information: + # + # + The process is in the foreground process group of its control terminal. + # < The process has raised CPU scheduling priority. + # > The process has specified a soft limit on memory requirements and is currently + # exceeding that limit; such a process is (necessarily) not swapped. + # A the process has asked for random page replacement + # (VA_ANOM, from vadvise(2), for example, lisp(1) in a garbage collect). + # E The process is trying to exit. + # L The process has pages locked in core (for example, for raw I/O). + # N The process has reduced CPU scheduling priority (see setpriority(2)). + # S The process has asked for FIFO page replacement (VA_SEQL, + # from vadvise(2), for example, a large image processing + # program using virtual memory to sequentially address + # voluminous data). + # s The process is a session leader. + # V The process is suspended during a vfork. + # W The process is swapped out. + # X The process is being traced or debugged. + + ch1 = proc_status[0] + + if ch1 == "Z": return True - if isinstance(e, ExecUtilException): - if e.exit_code == 2: - return True + if ch1 in "DIRSTU": + return False - return False + return None T_PID_TO_PPID = typing.Dict[int, int] From 896888681fbc9253de4b02dfa5129b5d11cf4344 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 22 Sep 2026 14:47:23 +0300 Subject: [PATCH 16/16] CI: (macos-14) matrix is rebuilt --- .github/workflows/ci.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 397f5cbd..2121fe92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,10 +295,27 @@ jobs: strategy: fail-fast: false matrix: - # Testing stable versions of Python on Apple Silicon (M1) architecture - python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] - # Primary target version of PostgreSQL for the native environment - postgres: ["17"] + include: + - python: "3.9" + postgres: "17" + case_suffix: "py3_09_xx-pg17_xx" + - python: "3.10" + postgres: "17" + case_suffix: "py3_10_xx-pg17_xx" + - python: "3.11" + postgres: "17" + case_suffix: "py3_11_xx-pg17_xx" + - python: "3.12" + postgres: "17" + case_suffix: "py3_12_xx-pg17_xx" + - python: "3.13" + postgres: "17" + case_suffix: "py3_13_xx-pg17_xx" + - python: "3.14" + postgres: "17" + case_suffix: "py3_14_xx-pg17_xx" + + name: "test: macos-14 | ${{ matrix.case_suffix }}" env: BASE_SIGN: "macos-py${{ matrix.python }}-pg${{ matrix.postgres }}"