Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions .github/scripts/ci-policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@
"spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"],
"codeql-analysis.yml": []},
}
# Exact maintenance inputs have no product consumers; their checks run in the planner.
MAINTENANCE_INPUTS = {
".github/PULL_REQUEST_TEMPLATE.md": [],
".github/dependabot.yml": [],
".github/scripts/check-rerun.py": [],
".github/scripts/test_check_rerun.py": [],
".github/scripts/check-docker-images.sh": ["docker"],
".github/scripts/docker-deployment.py": ["docker"],
".github/scripts/test_docker_deployment.py": ["docker"],
}
DEPENDENTS = {
"server": {"commons": ["server", "pd", "store", "hstore", "cluster"],
"struct": ["server", "pd", "store", "hstore", "cluster"],
Expand Down Expand Up @@ -96,6 +106,9 @@ def select(project, paths):
for path in paths:
if documentation(path):
continue
if project == "server" and path in MAINTENANCE_INPUTS:
selected.update(MAINTENANCE_INPUTS[path])
continue
if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")):
selected.add("dependency_license")
if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]:
Expand All @@ -110,7 +123,7 @@ def select(project, paths):
selected.update(["client", "go"])
continue
if project == "server":
if path == "hugegraph-server/hugegraph-api/pom.xml":
if path in {"hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"}:
selected.add("docker")
if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")):
selected.add("docker")
Expand Down Expand Up @@ -189,8 +202,11 @@ def create_plan(project, event, repository, fetch=api):
if not all(isinstance(plan[key], str) and plan[key] for key in ("source", "base", "head", "branch")):
raise StaleInputError("PR event has an empty input identity")
parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split()
if parents != [plan["base"], plan["head"]]:
if (plan["testedMergeSHA"] != os.environ.get("GITHUB_SHA")
or len(parents) != 2 or parents[1] != plan["head"]):
raise StaleInputError("checkout is not the event PR merge; start a new PR run")
# The event base may lag the synthetic merge after the target branch advances.
plan["base"] = parents[0]
require_current_pr(plan, fetch)
# head/base objects must exist locally; workflow fetches both before planning.
ancestor = git("merge-base", plan["base"], plan["head"])
Expand Down
44 changes: 39 additions & 5 deletions .github/scripts/docker-deployment.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import sys
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET


ADMIN_PASSWORD = "ci-compose-password"
Expand Down Expand Up @@ -59,6 +60,33 @@ def expect_response(url, expected=200, credentials=None):
return body


def expected_versions():
root = Path(__file__).resolve().parents[2]
ns = {"m": "http://maven.apache.org/POM/4.0.0"}
revision = ET.parse(root / "pom.xml").findtext("m:properties/m:revision", namespaces=ns)
gremlin = ET.parse(root / "hugegraph-server/pom.xml").findtext(
"m:properties/m:tinkerpop.version", namespaces=ns)
properties = dict(line.split("=", 1) for line in
(root / "hugegraph-commons/hugegraph-common/src/main/resources/version.properties")
.read_text().splitlines() if "=" in line and not line.startswith("#"))
if properties["VersionInBash"] != revision:
raise RuntimeError("VersionInBash does not match the project revision")
# Packaged API classes use their manifest version before the resource fallback.
api = ET.parse(root / "hugegraph-server/hugegraph-api/pom.xml").findtext(
".//m:manifestEntries/m:Implementation-Version", namespaces=ns) or properties["ApiVersion"]
expected = {"version": "v1", "core": revision, "gremlin": gremlin, "api": api}
if any(not isinstance(value, str) or not value for value in expected.values()):
raise RuntimeError("Source POMs did not define non-empty expected versions")
return expected


def verify_versions(payload):
versions = payload.get("versions") if isinstance(payload, dict) else None
expected = expected_versions()
if not isinstance(versions, dict) or any(versions.get(key) != value for key, value in expected.items()):
raise RuntimeError(f"Server did not return the expected versions object: {expected}; got {versions}")


def verify_server():
url = "http://localhost:8080"
expect_response(url + "/graphspaces/DEFAULT/graphs", 401)
Expand All @@ -68,11 +96,7 @@ def verify_server():
if not isinstance(names, list) or "hugegraph" not in names:
raise RuntimeError("Server did not return its initialized hugegraph in the graphs array")
payload = json.loads(expect_response(url + "/versions"))
versions = payload.get("versions") if isinstance(payload, dict) else None
if not isinstance(versions, dict) or versions.get("version") != "v1" or any(
not isinstance(versions.get(key), str) or not versions[key]
for key in ("core", "gremlin", "api")):
raise RuntimeError("Server did not return the expected versions object")
verify_versions(payload)


def verify_storage():
Expand All @@ -91,6 +115,15 @@ def verify_storage():
raise RuntimeError("Authenticated PD did not return its registered stores array")


def verify_graph(run_id):
root = Path(__file__).resolve().parents[2]
script = root / "hugegraph-server/hugegraph-dist/src/assembly/travis/run-server-e2e-smoke-test.sh"
environment = dict(os.environ, HUGEGRAPH_USERNAME="admin", HUGEGRAPH_PASSWORD=ADMIN_PASSWORD)
# Reuse the write/read/Gremlin assertions; allow its readiness and bounded HTTP requests.
subprocess.run(["bash", str(script), "http://localhost:8080", "create", run_id],
check=True, timeout=960, env=environment)


def smoke(tag, topology, images):
root = Path(__file__).resolve().parents[2]
project = f"hg-pr-{tag}-{Path(topology).stem}"
Expand All @@ -113,6 +146,7 @@ def smoke(tag, topology, images):
verify_server()
if "pd" in images:
verify_storage()
verify_graph(project.replace("-", "_"))
except BaseException:
failed = True
for args in (["ps"], ["logs", "--no-color", "--tail", "200"]):
Expand Down
44 changes: 42 additions & 2 deletions .github/scripts/test_ci_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ def live_pr(self):
"base": {"sha": "base", "repo": {"full_name": "apache/server"}}}

def setUp(self):
event_sha = patch.dict(os.environ, {"GITHUB_SHA": "merge"})
self.addCleanup(event_sha.stop)
event_sha.start()
# All tests stay local; successful PR gates query only this current-PR fixture.
api_mock = patch.object(policy, "api", return_value=self.live_pr())
self.addCleanup(api_mock.stop)
Expand Down Expand Up @@ -110,6 +113,22 @@ def test_single_workflow_has_no_global_fanout(self):
self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"]))
self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"]))

def test_known_maintenance_inputs_have_specific_owners(self):
for path in [".github/PULL_REQUEST_TEMPLATE.md", ".github/dependabot.yml",
".github/scripts/check-rerun.py", ".github/scripts/test_check_rerun.py"]:
with self.subTest(path=path):
self.assertEqual(set(), policy.select("server", [path]))
for path in [".github/scripts/check-docker-images.sh", ".github/scripts/docker-deployment.py",
".github/scripts/test_docker_deployment.py"]:
with self.subTest(path=path):
self.assertEqual({"docker"}, policy.select("server", [path]))
selected = policy.select("server", [path, "hugegraph-pd/hg-pd-core/src/A.java"])
self.assertTrue({"docker", "pd", "store", "hstore", "cluster"}.issubset(selected))
for path in [".github/scripts/ci-policy.py", ".github/scripts/test_ci_policy.py",
".github/workflows/rerun-ci.yml", ".github/dependabot-unknown.yml"]:
with self.subTest(path=path):
self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path]))

def test_unknown_and_proto_fail_conservative(self):
for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]:
self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path]))
Expand Down Expand Up @@ -192,8 +211,10 @@ def test_memory_gate_ignores_advisory_cancellation(self):
self.assertEqual("cancelled", failure.exception.report["results"]["cluster"])
self.assertNotIn("server_memory", failure.exception.report["results"])

def test_api_pom_is_a_docker_input(self):
self.assertIn("docker", policy.select("server", ["hugegraph-server/hugegraph-api/pom.xml"]))
def test_server_and_api_poms_are_docker_inputs(self):
for path in ["hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"]:
with self.subTest(path=path):
self.assertIn("docker", policy.select("server", [path]))
self.assertNotIn("docker", policy.select("server", ["hugegraph-server/hugegraph-api/src/main/A.java"]))

def test_codeql_and_smoke_follow_affected_inputs(self):
Expand Down Expand Up @@ -359,7 +380,12 @@ def fetch(path):
old = os.getcwd()
try:
os.chdir(root)
os.environ["GITHUB_SHA"] = merge
plan = policy.create_plan("server", event, "apache/server", fetch)
git("checkout", "--detach", "-q", false_merge)
with self.assertRaises(policy.StaleInputError):
policy.create_plan("server", event, "apache/server", fetch)
git("checkout", "--detach", "-q", merge)
finally:
os.chdir(old)
self.assertEqual(["repos/apache/server/pulls/7"], calls)
Expand Down Expand Up @@ -446,6 +472,7 @@ def git(*args):
old = os.getcwd()
try:
os.chdir(root)
os.environ["GITHUB_SHA"] = merge
plan = policy.create_plan("server", event, "apache/server", lambda _: live)
results = {suite: {"result": "success"} for suite in plan["expected"]}
results.update(plan={"result": "success"}, fixture={"result": "success"},
Expand All @@ -461,6 +488,19 @@ def git(*args):
policy.create_plan("server", event, "apache/server", lambda _: changed)
with self.assertRaises(policy.StaleInputError):
policy.gate(plan, results, lambda _: changed)
advanced_merge = git("commit-tree", tree, "-p", advanced_base, "-p", head,
"-m", "PR merge after master advanced")
git("checkout", "--detach", "-q", advanced_merge)
os.environ["GITHUB_SHA"] = advanced_merge
advanced_plan = policy.create_plan("server", event, "apache/server", lambda _: live)
self.assertEqual(base, event["pull_request"]["base"]["sha"])
self.assertEqual(advanced_base, advanced_plan["base"])
self.assertEqual(advanced_merge, advanced_plan["testedMergeSHA"])
self.assertEqual(plan["changedPaths"], advanced_plan["changedPaths"])
policy.gate(advanced_plan, results, lambda _: live)
with patch.dict(os.environ, {"GITHUB_SHA": ""}):
with self.assertRaises(policy.StaleInputError):
policy.create_plan("server", event, "apache/server", lambda _: live)
with self.assertRaises(policy.StaleInputError):
policy.gate(plan, results, lambda _: dict(live, state="closed"))
git("checkout", "--detach", "-q", head)
Expand Down
66 changes: 64 additions & 2 deletions .github/scripts/test_docker_deployment.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
"""Reject successful service checks performed against an unrelated image."""

import importlib.util
import json
import os
from pathlib import Path
import unittest
from unittest.mock import patch
Expand Down Expand Up @@ -73,18 +75,58 @@ def test_hstore_starts_only_pr_services_and_cleans_after_checks(self):
with patch.object(deployment, "command", side_effect=identities + containers), \
patch.object(deployment.subprocess, "run") as run, \
patch.object(deployment, "verify_server") as server, \
patch.object(deployment, "verify_storage") as storage:
patch.object(deployment, "verify_storage") as storage, \
patch.object(deployment, "verify_graph") as graph:
deployment.smoke("ci-1-1", "docker-compose-hstore.yml", images)
startup = run.call_args_list[0].args[0]
self.assertEqual(startup[-3:], ["pd", "store", "server"])
self.assertNotIn("hubble", startup)
server.assert_called_once_with()
storage.assert_called_once_with()
graph.assert_called_once_with("hg_pr_ci_1_1_docker_compose_hstore")
self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"])


class GraphSmokeTest(unittest.TestCase):
def test_reuses_graph_write_read_and_gremlin_checks_with_auth(self):
with patch.dict(os.environ, {"PATH": "/existing/tools"}), \
patch.object(deployment.subprocess, "run") as run:
deployment.verify_graph("ci_1_1")
invocation = run.call_args
self.assertTrue(invocation.args[0][1].endswith("run-server-e2e-smoke-test.sh"))
self.assertEqual(invocation.args[0][2:], ["http://localhost:8080", "create", "ci_1_1"])
self.assertTrue(invocation.kwargs["check"])
self.assertEqual(invocation.kwargs["timeout"], 960)
environment = invocation.kwargs["env"]
self.assertEqual(environment["PATH"], "/existing/tools")
self.assertEqual(environment["HUGEGRAPH_USERNAME"], "admin")
self.assertEqual(environment["HUGEGRAPH_PASSWORD"], deployment.ADMIN_PASSWORD)

def test_graph_failure_propagates_and_cleans_both_topologies(self):
for topology, images in (
("docker-compose.yml", {"server": "hugegraph/hugegraph"}),
("docker-compose-hstore.yml", {"pd": "hugegraph/pd", "store": "hugegraph/store",
"server": "hugegraph/server"})):
identities = [f"sha256:{service}" for service in images]
containers = [value for service in images for value in (service, f"sha256:{service}", "healthy")]
failure = deployment.subprocess.CalledProcessError(1, "graph smoke")
with self.subTest(topology=topology), \
patch.object(deployment, "command", side_effect=identities + containers), \
patch.object(deployment.subprocess, "run") as run, \
patch.object(deployment, "verify_server"), \
patch.object(deployment, "verify_storage"), \
patch.object(deployment, "verify_graph", side_effect=failure) as graph:
with self.assertRaises(deployment.subprocess.CalledProcessError) as caught:
deployment.smoke("ci-1-1", topology, images)
self.assertIs(caught.exception, failure)
graph.assert_called_once()
calls = [call.args[0] for call in run.call_args_list]
self.assertTrue(any("logs" in args for args in calls))
self.assertEqual(calls[-1][-3:], ["down", "-v", "--remove-orphans"])


class PayloadTest(unittest.TestCase):
VERSIONS = '{"versions":{"version":"v1","core":"1.7.0","gremlin":"3.7.3","api":"0.74"}}'
VERSIONS = json.dumps({"versions": deployment.expected_versions()})
GRAPHS = '{"graphs":["hugegraph"]}'

def test_accepts_public_versions_and_authenticated_graphs(self):
Expand Down Expand Up @@ -112,6 +154,26 @@ def test_rejects_invalid_versions_even_when_http_status_is_200(self):
with self.assertRaisesRegex(RuntimeError, "versions object"):
deployment.verify_server()

def test_rejects_wrong_release_gremlin_and_protocol_versions(self):
for key, wrong in (("core", "1.7.0"), ("gremlin", "3.7.3"), ("gremlin", None),
("api", "1.8.0"), ("version", "v2")):
payload = json.loads(self.VERSIONS)
payload["versions"][key] = wrong
with self.subTest(key=key), self.assertRaisesRegex(RuntimeError, "versions object"):
deployment.verify_versions(payload)

def test_rejects_missing_expected_gremlin_version(self):
parse = deployment.ET.parse

def read_pom(path):
if path == Path(deployment.__file__).resolve().parents[2] / "hugegraph-server/pom.xml":
return deployment.ET.ElementTree(deployment.ET.Element("project"))
return parse(path)

with patch.object(deployment.ET, "parse", side_effect=read_pom):
with self.assertRaisesRegex(RuntimeError, "non-empty expected versions"):
deployment.expected_versions()

def test_rejects_unauthenticated_server_access(self):
with patch.object(deployment, "response", return_value=(200, self.GRAPHS)):
with self.assertRaisesRegex(RuntimeError, "expected 401"):
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/cluster-test-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ permissions:

jobs:
cluster-test:
timeout-minutes: 120
runs-on: ubuntu-latest
env:
USE_STAGE: 'false' # Whether to include the stage repository.
Expand Down
22 changes: 3 additions & 19 deletions .github/workflows/commons-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ jobs:
contents: read

build-commons:
timeout-minutes: 20
name: Commons tests (Java ${{ matrix.JAVA_VERSION }})
needs: java-runtime
runs-on: ubuntu-latest
Expand Down Expand Up @@ -47,7 +48,8 @@ jobs:

- name: Install
run: |
mvn install -Dmaven.javadoc.skip=true -ntp -Dmaven.test.skip=true
mvn install -pl hugegraph-commons/hugegraph-common,hugegraph-commons/hugegraph-rpc -am \
-Dmaven.javadoc.skip=true -ntp -Dmaven.test.skip=true

- name: Run common test
run: |
Expand All @@ -66,21 +68,3 @@ jobs:
files: hugegraph-commons/target/jacoco.xml
disable_search: true
fail_ci_if_error: false

# Temporary compatibility check; Commons becomes optional when .asf.yaml takes effect.
legacy-commons-required:
name: build-commons (11)
needs: [ java-runtime, build-commons ]
if: ${{ always() }}
runs-on: ubuntu-24.04
permissions: {}
steps:
- name: Require the runtime configuration and all Commons tests to pass
env:
RUNTIME_RESULT: ${{ needs.java-runtime.result }}
COMMONS_RESULT: ${{ needs.build-commons.result }}
run: |
if [ "$RUNTIME_RESULT" != "success" ] || [ "$COMMONS_RESULT" != "success" ]; then
echo "::error::Runtime configuration: $RUNTIME_RESULT; Commons tests: $COMMONS_RESULT"
exit 1
fi
2 changes: 1 addition & 1 deletion .github/workflows/docker-build-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,5 +128,5 @@ jobs:
- name: Verify deployment checker contracts
run: python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py'

- name: Start the exact PR images and verify health and authentication
- name: Verify exact PR images, authentication and graph read/write
run: python3 .github/scripts/docker-deployment.py "$IMAGE_TAG"
Loading
Loading