feat: enable ginkgo parallel execution for API7EE E2E tests (#404)

Author: AlinsRan

.github/workflows/e2e-test.yml

@@ -61,6 +61,10 @@ jobs:
           curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
           chmod 700 get_helm.sh
           ./get_helm.sh
+
+      - name: Install ginkgo
+        run: make install-ginkgo
+
       - name: Login to Registry
         uses: docker/login-action@v3
         with:
@@ -121,4 +125,8 @@ jobs:
           TEST_LABEL: ${{ matrix.cases_subset }}
           TEST_ENV: CI
         run: |
-          make e2e-test
+          if [[ "${{ matrix.cases_subset }}" == "webhook" ]]; then
+            E2E_NODES=1 make ginkgo-api7ee-e2e-test
+          else
+            E2E_NODES=4 make ginkgo-api7ee-e2e-test
+          fi

Makefile

@@ -34,7 +34,7 @@ ADC_VERSION ?= 0.22.1
 
 DIR := $(shell pwd)
 
-GINKGO_VERSION ?= 2.20.0
+GINKGO_VERSION ?= 2.22.0
 TEST_TIMEOUT ?= 80m
 TEST_DIR ?= ./test/e2e/
 E2E_NODES ?= 4
@@ -154,6 +154,13 @@ download-api7ee3-chart:
 ginkgo-e2e-test: adc
 	@ginkgo -cover -coverprofile=coverage.txt -r --randomize-all --randomize-suites --trace --focus=$(E2E_FOCUS) --nodes=$(E2E_NODES) --label-filter="$(TEST_LABEL)" $(TEST_DIR)
 
+.PHONY: ginkgo-api7ee-e2e-test
+ginkgo-api7ee-e2e-test: adc
+	@DASHBOARD_VERSION=$(DASHBOARD_VERSION) ginkgo -cover -coverprofile=coverage.txt \
+		--randomize-all --randomize-suites --trace \
+		--timeout=$(TEST_TIMEOUT) --nodes=$(E2E_NODES) \
+		--label-filter="$(TEST_LABEL)" ./test/e2e/
+
 .PHONY: install-ginkgo
 install-ginkgo:
 	@go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION)

test/e2e/e2e_test.go

@@ -42,8 +42,13 @@ func TestE2E(t *testing.T) {
 	// init newDeployer function
 	scaffold.NewDeployer = scaffold.NewAPI7Deployer
 
-	BeforeSuite(f.BeforeSuite)
-	AfterSuite(f.AfterSuite)
+	// DeployAPI7EE runs only on ginkgo node 1 and deploys the shared API7EE control plane.
+	// InitNodeConnections runs on every node to set up per-node dashboard connections.
+	SynchronizedBeforeSuite(f.DeployAPI7EE, f.InitNodeConnections)
+
+	// CloseNodeConnections runs on every node to close per-node dashboard tunnels.
+	// TeardownInfrastructure runs only on node 1 for any suite-level cleanup.
+	SynchronizedAfterSuite(f.CloseNodeConnections, f.TeardownInfrastructure)
 
 	_, _ = fmt.Fprintf(GinkgoWriter, "Starting apisix-ingress suite\n")
 	RunSpecs(t, "e2e suite")

test/e2e/framework/api7_framework.go

@@ -37,23 +37,30 @@ import (
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
+const defaultDashboardVersion = "dev"
+
 var (
 	API7EELicense string
 
 	dashboardVersion string
 )
 
-func (f *Framework) BeforeSuite() {
-	// init license and dashboard version
+// initSuiteEnv reads required environment variables and panics early with a clear
+// message if any mandatory variable is missing.
+func initSuiteEnv() {
 	API7EELicense = os.Getenv("API7_EE_LICENSE")
 	if API7EELicense == "" {
 		panic("env {API7_EE_LICENSE} is required")
 	}
 
 	dashboardVersion = os.Getenv("DASHBOARD_VERSION")
 	if dashboardVersion == "" {
-		dashboardVersion = "dev"
+		dashboardVersion = defaultDashboardVersion
 	}
+}
+
+func (f *Framework) BeforeSuite() {
+	initSuiteEnv()
 
 	_ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
 
@@ -83,6 +90,61 @@ func (f *Framework) AfterSuite() {
 	f.shutdownDashboardTunnel()
 }
 
+// DeployAPI7EE deploys the API7EE control plane once (runs on ginkgo node 1 only).
+// It returns a ready signal consumed by InitNodeConnections on all nodes.
+func (f *Framework) DeployAPI7EE() []byte {
+	initSuiteEnv()
+
+	_ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
+
+	Eventually(func() error {
+		_, err := k8s.GetNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("namespace %s still exists", _namespace)
+	}, "1m", "2s").Should(Succeed())
+
+	k8s.CreateNamespace(GinkgoT(), f.kubectlOpts, _namespace)
+
+	f.DeployComponents()
+
+	time.Sleep(1 * time.Minute)
+
+	// Create a temporary tunnel for one-time setup operations.
+	// Each node will create its own persistent tunnel in InitNodeConnections.
+	err := f.newDashboardTunnel()
+	Expect(err).ShouldNot(HaveOccurred(), "creating temporary dashboard tunnel")
+	f.Logf("Temporary dashboard tunnel: %s", _dashboardHTTPTunnel.Endpoint())
+
+	f.UploadLicense()
+	f.setDpManagerEndpoints()
+
+	// Close the temporary tunnel; each node creates its own in InitNodeConnections.
+	f.shutdownDashboardTunnel()
+
+	return []byte("ready")
+}
+
+// InitNodeConnections initializes per-node connections to the shared API7EE control plane.
+// It runs on every ginkgo parallel node after DeployAPI7EE completes.
+func (f *Framework) InitNodeConnections(_ []byte) {
+	initSuiteEnv()
+
+	err := f.newDashboardTunnel()
+	Expect(err).ShouldNot(HaveOccurred(), "creating dashboard tunnel for node")
+	f.Logf("Dashboard HTTP Tunnel: %s", _dashboardHTTPTunnel.Endpoint())
+}
+
+// CloseNodeConnections closes per-node connections. Runs on every ginkgo parallel node.
+func (f *Framework) CloseNodeConnections() {
+	f.shutdownDashboardTunnel()
+}
+
+// TeardownInfrastructure cleans up suite-level resources. Runs on ginkgo node 1 only.
+// The Kind cluster is deleted by CI after the job, so this is a no-op.
+func (f *Framework) TeardownInfrastructure() {}
+
 // DeployComponents deploy necessary components
 func (f *Framework) DeployComponents() {
 	f.deploy()
@@ -167,31 +229,43 @@ var (
 	_dashboardHTTPSTunnel *k8s.Tunnel
 )
 
+// dashboardLocalPorts returns the local port pair to use for the dashboard HTTP
+// and HTTPS tunnels.  Each ginkgo parallel process gets a unique, non-overlapping
+// range based on its 1-indexed process number, eliminating port conflicts without
+// any TOCTOU race.
+//
+// Formula: base = 18000 + node*100
+//
+//	node=1 → 18100 (HTTP) / 18101 (HTTPS)
+//	node=2 → 18200 (HTTP) / 18201 (HTTPS)
+func dashboardLocalPorts() (httpLocal, httpsLocal int) {
+	node := GinkgoParallelProcess() // 1-indexed
+	base := 18000 + node*100
+	return base, base + 1
+}
+
 func (f *Framework) newDashboardTunnel() error {
 	var (
-		httpNodePort  int
-		httpsNodePort int
-		httpPort      int
-		httpsPort     int
+		httpPort  int
+		httpsPort int
 	)
 
 	service := k8s.GetService(f.GinkgoT, f.kubectlOpts, "api7ee3-dashboard")
 
 	for _, port := range service.Spec.Ports {
 		switch port.Name {
 		case "http":
-			httpNodePort = int(port.NodePort)
 			httpPort = int(port.Port)
 		case "https":
-			httpsNodePort = int(port.NodePort)
 			httpsPort = int(port.Port)
 		}
 	}
 
+	httpLocal, httpsLocal := dashboardLocalPorts()
 	_dashboardHTTPTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
-		httpNodePort, httpPort)
+		httpLocal, httpPort)
 	_dashboardHTTPSTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
-		httpsNodePort, httpsPort)
+		httpsLocal, httpsPort)
 
 	if err := _dashboardHTTPTunnel.ForwardPortE(f.GinkgoT); err != nil {
 		return err