diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index e512d4b6..807d30a5 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -61,6 +61,10 @@ jobs: curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 chmod 700 get_helm.sh ./get_helm.sh + + - name: Install ginkgo + run: make install-ginkgo + - name: Login to Registry uses: docker/login-action@v3 with: @@ -121,4 +125,8 @@ jobs: TEST_LABEL: ${{ matrix.cases_subset }} TEST_ENV: CI run: | - make e2e-test + if [[ "${{ matrix.cases_subset }}" == "webhook" ]]; then + E2E_NODES=1 make ginkgo-api7ee-e2e-test + else + E2E_NODES=4 make ginkgo-api7ee-e2e-test + fi diff --git a/Makefile b/Makefile index 76ff2d5e..51fbe472 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ ADC_VERSION ?= 0.22.1 DIR := $(shell pwd) -GINKGO_VERSION ?= 2.20.0 +GINKGO_VERSION ?= 2.22.0 TEST_TIMEOUT ?= 80m TEST_DIR ?= ./test/e2e/ E2E_NODES ?= 4 @@ -154,6 +154,13 @@ download-api7ee3-chart: ginkgo-e2e-test: adc @ginkgo -cover -coverprofile=coverage.txt -r --randomize-all --randomize-suites --trace --focus=$(E2E_FOCUS) --nodes=$(E2E_NODES) --label-filter="$(TEST_LABEL)" $(TEST_DIR) +.PHONY: ginkgo-api7ee-e2e-test +ginkgo-api7ee-e2e-test: adc + @DASHBOARD_VERSION=$(DASHBOARD_VERSION) ginkgo -cover -coverprofile=coverage.txt \ + --randomize-all --randomize-suites --trace \ + --timeout=$(TEST_TIMEOUT) --nodes=$(E2E_NODES) \ + --label-filter="$(TEST_LABEL)" ./test/e2e/ + .PHONY: install-ginkgo install-ginkgo: @go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index fbc307b6..746500e2 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -42,8 +42,13 @@ func TestE2E(t *testing.T) { // init newDeployer function scaffold.NewDeployer = scaffold.NewAPI7Deployer - BeforeSuite(f.BeforeSuite) - AfterSuite(f.AfterSuite) + // DeployAPI7EE runs only on ginkgo node 1 and deploys the shared API7EE control plane. + // InitNodeConnections runs on every node to set up per-node dashboard connections. + SynchronizedBeforeSuite(f.DeployAPI7EE, f.InitNodeConnections) + + // CloseNodeConnections runs on every node to close per-node dashboard tunnels. + // TeardownInfrastructure runs only on node 1 for any suite-level cleanup. + SynchronizedAfterSuite(f.CloseNodeConnections, f.TeardownInfrastructure) _, _ = fmt.Fprintf(GinkgoWriter, "Starting apisix-ingress suite\n") RunSpecs(t, "e2e suite") diff --git a/test/e2e/framework/api7_framework.go b/test/e2e/framework/api7_framework.go index 793b76f3..b4d2b43b 100644 --- a/test/e2e/framework/api7_framework.go +++ b/test/e2e/framework/api7_framework.go @@ -37,14 +37,17 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" ) +const defaultDashboardVersion = "dev" + var ( API7EELicense string dashboardVersion string ) -func (f *Framework) BeforeSuite() { - // init license and dashboard version +// initSuiteEnv reads required environment variables and panics early with a clear +// message if any mandatory variable is missing. +func initSuiteEnv() { API7EELicense = os.Getenv("API7_EE_LICENSE") if API7EELicense == "" { panic("env {API7_EE_LICENSE} is required") @@ -52,8 +55,12 @@ func (f *Framework) BeforeSuite() { dashboardVersion = os.Getenv("DASHBOARD_VERSION") if dashboardVersion == "" { - dashboardVersion = "dev" + dashboardVersion = defaultDashboardVersion } +} + +func (f *Framework) BeforeSuite() { + initSuiteEnv() _ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace) @@ -83,6 +90,61 @@ func (f *Framework) AfterSuite() { f.shutdownDashboardTunnel() } +// DeployAPI7EE deploys the API7EE control plane once (runs on ginkgo node 1 only). +// It returns a ready signal consumed by InitNodeConnections on all nodes. +func (f *Framework) DeployAPI7EE() []byte { + initSuiteEnv() + + _ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace) + + Eventually(func() error { + _, err := k8s.GetNamespaceE(GinkgoT(), f.kubectlOpts, _namespace) + if k8serrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("namespace %s still exists", _namespace) + }, "1m", "2s").Should(Succeed()) + + k8s.CreateNamespace(GinkgoT(), f.kubectlOpts, _namespace) + + f.DeployComponents() + + time.Sleep(1 * time.Minute) + + // Create a temporary tunnel for one-time setup operations. + // Each node will create its own persistent tunnel in InitNodeConnections. + err := f.newDashboardTunnel() + Expect(err).ShouldNot(HaveOccurred(), "creating temporary dashboard tunnel") + f.Logf("Temporary dashboard tunnel: %s", _dashboardHTTPTunnel.Endpoint()) + + f.UploadLicense() + f.setDpManagerEndpoints() + + // Close the temporary tunnel; each node creates its own in InitNodeConnections. + f.shutdownDashboardTunnel() + + return []byte("ready") +} + +// InitNodeConnections initializes per-node connections to the shared API7EE control plane. +// It runs on every ginkgo parallel node after DeployAPI7EE completes. +func (f *Framework) InitNodeConnections(_ []byte) { + initSuiteEnv() + + err := f.newDashboardTunnel() + Expect(err).ShouldNot(HaveOccurred(), "creating dashboard tunnel for node") + f.Logf("Dashboard HTTP Tunnel: %s", _dashboardHTTPTunnel.Endpoint()) +} + +// CloseNodeConnections closes per-node connections. Runs on every ginkgo parallel node. +func (f *Framework) CloseNodeConnections() { + f.shutdownDashboardTunnel() +} + +// TeardownInfrastructure cleans up suite-level resources. Runs on ginkgo node 1 only. +// The Kind cluster is deleted by CI after the job, so this is a no-op. +func (f *Framework) TeardownInfrastructure() {} + // DeployComponents deploy necessary components func (f *Framework) DeployComponents() { f.deploy() @@ -167,12 +229,25 @@ var ( _dashboardHTTPSTunnel *k8s.Tunnel ) +// dashboardLocalPorts returns the local port pair to use for the dashboard HTTP +// and HTTPS tunnels. Each ginkgo parallel process gets a unique, non-overlapping +// range based on its 1-indexed process number, eliminating port conflicts without +// any TOCTOU race. +// +// Formula: base = 18000 + node*100 +// +// node=1 → 18100 (HTTP) / 18101 (HTTPS) +// node=2 → 18200 (HTTP) / 18201 (HTTPS) +func dashboardLocalPorts() (httpLocal, httpsLocal int) { + node := GinkgoParallelProcess() // 1-indexed + base := 18000 + node*100 + return base, base + 1 +} + func (f *Framework) newDashboardTunnel() error { var ( - httpNodePort int - httpsNodePort int - httpPort int - httpsPort int + httpPort int + httpsPort int ) service := k8s.GetService(f.GinkgoT, f.kubectlOpts, "api7ee3-dashboard") @@ -180,18 +255,17 @@ func (f *Framework) newDashboardTunnel() error { for _, port := range service.Spec.Ports { switch port.Name { case "http": - httpNodePort = int(port.NodePort) httpPort = int(port.Port) case "https": - httpsNodePort = int(port.NodePort) httpsPort = int(port.Port) } } + httpLocal, httpsLocal := dashboardLocalPorts() _dashboardHTTPTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard", - httpNodePort, httpPort) + httpLocal, httpPort) _dashboardHTTPSTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard", - httpsNodePort, httpsPort) + httpsLocal, httpsPort) if err := _dashboardHTTPTunnel.ForwardPortE(f.GinkgoT); err != nil { return err