Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
branches: [main]

env:
BINK_COMMIT: 2b1bbdca74fc36d319fb1ea6d0523d9a732fcb1f
BINK_COMMIT: fdfdc863d9f4e9caf83e56c06bf7f940cfb9f315

permissions: {}

Expand Down
31 changes: 26 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ CONTAINER_TOOL ?= podman
BINK_CLUSTER_NAME ?= e2e
KUBECONFIG_BINK ?= ./kubeconfig-$(BINK_CLUSTER_NAME)
ARTIFACTS ?= $(abspath _output/logs)
BINK_NODE_DISK_IMAGE ?= ghcr.io/alicefr/bink/node:v1.35-fedora-44-disk
BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node
# YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header.
YEAR ?= $(shell date +%Y)

Expand Down Expand Up @@ -62,7 +64,10 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
rm -rf $(ARTIFACTS)
cd test/e2e && KUBECONFIG=$(abspath $(KUBECONFIG_BINK)) BINK_CLUSTER_NAME=$(BINK_CLUSTER_NAME) \
$(if $(BINK_NODE_IMAGE),BINK_NODE_IMAGE=$(BINK_NODE_IMAGE)) \
BINK_NODE_DISK_IMAGE=$(BINK_NODE_DISK_IMAGE) \
BINK_LOCAL_REGISTRY_NODE_IMAGE=$(BINK_LOCAL_REGISTRY_NODE_IMAGE) \
ARTIFACTS=$(ARTIFACTS) \
BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \
go test -timeout 10m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) .

##@ Build
Expand All @@ -82,6 +87,12 @@ build-daemon: ## Build daemon binary.
buildimg: ## Build container image.
$(CONTAINER_TOOL) build -t $(IMG) .

.PHONY: build-update-image
build-update-image: ## Build a derived node image for update testing and push to bink registry.
@printf 'FROM localhost:5000/node:latest\nRUN touch /usr/share/update-marker\n' | \
podman build -t localhost:5000/node:update -f - .
podman push --tls-verify=false localhost:5000/node:update
Comment thread
jlebon marked this conversation as resolved.

##@ Deployment

ifndef ignore-not-found
Expand Down Expand Up @@ -111,16 +122,26 @@ undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.
# Note the :latest tag here: this makes the pull policy be Always.
IMG_BINK ?= registry.cluster.local:5000/bootc-operator-e2e:latest

.PHONY: start-bink
start-bink: ## Start a bink cluster (idempotent).
.PHONY: seed-node-image
seed-node-image: ## Pull the bootc node image by digest and push to the bink registry.
bink registry start
bink cluster list 2>&1 | grep -qw $(BINK_CLUSTER_NAME) || \
podman pull $(BINK_NODE_DISK_IMAGE)
bootc_img=$$(podman inspect --format '{{index .Config.Labels "bink.bootc-image"}}' $(BINK_NODE_DISK_IMAGE)) && \
bootc_digest=$$(podman inspect --format '{{index .Config.Labels "bink.bootc-image-digest"}}' $(BINK_NODE_DISK_IMAGE)) && \
podman pull "$$bootc_img@$$bootc_digest" && \
podman tag "$$bootc_img@$$bootc_digest" localhost:5000/node:latest
podman push --tls-verify=false localhost:5000/node:latest

.PHONY: start-bink
start-bink: seed-node-image ## Start a bink cluster (idempotent).
bink cluster list 2>&1 | grep -qw $(BINK_CLUSTER_NAME) || { \
node_digest=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) && \
bink cluster start --cluster-name $(BINK_CLUSTER_NAME) --node-name controller --api-port 0 --expose $(KUBECONFIG_BINK) \
$(if $(BINK_NODE_IMAGE),--node-image $(BINK_NODE_IMAGE))
--node-image $(BINK_NODE_DISK_IMAGE) --target-imgref $(BINK_LOCAL_REGISTRY_NODE_IMAGE)@$$node_digest; }
kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m

.PHONY: deploy-bink
deploy-bink: start-bink kustomize ## Deploy to a bink cluster (requires: buildimg).
deploy-bink: start-bink build-update-image kustomize ## Deploy to a bink cluster (requires: buildimg).
podman push --tls-verify=false $(IMG) localhost:5000/bootc-operator-e2e:latest
# On re-deploy, restart the rollout to force a re-pull of the :latest tag.
# On fresh deploy, skip the restart -- the pod is already pulling the correct image.
Comment thread
alicefr marked this conversation as resolved.
Expand Down
13 changes: 6 additions & 7 deletions test/e2e/bootcnode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (

bootcv1alpha1 "github.com/jlebon/bootc-operator/api/v1alpha1"
"github.com/jlebon/bootc-operator/test/e2e/e2eutil"
testutil "github.com/jlebon/bootc-operator/test/util"
)

const (
Expand All @@ -35,9 +34,7 @@ func TestControllerMembership(t *testing.T) {

ctx := context.Background()

// Create a pool selecting this test's nodes.
imageRef := testutil.ImageDigestRefA
pool := env.NewPool("workers", imageRef)
pool := env.NewPool("workers", env.NodeImageDigestedPullSpec())
g.Expect(env.Client.Create(ctx, pool)).To(Succeed())

// Wait for BootcNode to appear for the worker.
Expand All @@ -52,7 +49,7 @@ func TestControllerMembership(t *testing.T) {
g.Expect(owner.Name).To(Equal(pool.Name))

// Verify desiredImage.
g.Expect(bn.Spec.DesiredImage).To(Equal(imageRef))
g.Expect(bn.Spec.DesiredImage).To(Equal(env.NodeImageDigestedPullSpec()))

// Verify the worker has the managed label.
var node corev1.Node
Expand All @@ -79,8 +76,10 @@ func TestControllerMembership(t *testing.T) {
g.Eventually(func(g Gomega) {
g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed())
g.Expect(bn.Status.Booted).NotTo(BeNil(), "expected booted status to be populated")
g.Expect(bn.Status.Booted.Image).NotTo(BeEmpty(), "expected booted image to be non-empty")
g.Expect(bn.Status.Booted.ImageDigest).NotTo(BeEmpty(), "expected booted imageDigest to be non-empty")
g.Expect(bn.Status.Booted.Image).To(Equal(env.NodeImageDigestedPullSpec()),
"booted image should match seeded registry image")
g.Expect(bn.Status.Booted.ImageDigest).To(Equal(env.NodeImageDigest()),
"booted image digest should match seeded registry image")
g.Expect(bn.Status.Conditions).To(ContainElement(And(
HaveField("Type", bootcv1alpha1.NodeIdle),
HaveField("Status", metav1.ConditionTrue),
Expand Down
57 changes: 51 additions & 6 deletions test/e2e/e2eutil/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ type Env struct {

// nodes tracks node names added via AddNode for cleanup.
nodes []string

// nodeImageDigest is the manifest digest of the bootc image seeded
// into the bink registry (e.g. "sha256:abc123..."). Empty when not seeded.
nodeImageDigest string

// nodeImageRegistry is the in-cluster registry path for the seeded node image
// (e.g. "registry.cluster.local:5000/node"). Empty when not seeded.
nodeImageRegistry string
}

// New connects to an existing bink cluster and returns an Env ready
Expand All @@ -70,12 +78,17 @@ func New(t *testing.T) *Env {
t.Fatal("BINK_CLUSTER_NAME must be set")
}

nodeImageDigest := os.Getenv("BINK_NODE_IMAGE_DIGEST")
nodeImageRegistry := os.Getenv("BINK_LOCAL_REGISTRY_NODE_IMAGE")

k8sClient := buildClient(t, kubeconfigPath)

env := &Env{
Client: k8sClient,
clusterName: clusterName,
testID: sanitizeTestName(t.Name()),
Client: k8sClient,
clusterName: clusterName,
testID: sanitizeTestName(t.Name()),
nodeImageDigest: nodeImageDigest,
nodeImageRegistry: nodeImageRegistry,
}

t.Cleanup(func() {
Expand All @@ -89,8 +102,9 @@ func New(t *testing.T) *Env {
type NodeOption func(*nodeConfig)

type nodeConfig struct {
memory int
labels map[string]string
memory int
labels map[string]string
targetImgRef string
}

// WithMemory sets the VM memory in MB for the node.
Expand All @@ -111,6 +125,15 @@ func WithLabel(key, value string) NodeOption {
}
}

// WithTargetImgRef sets the target image reference for the node,
// passed as --target-imgref to bink node add. Overrides the automatic
// default that AddNode applies when registry metadata is available.
func WithTargetImgRef(ref string) NodeOption {
return func(c *nodeConfig) {
c.targetImgRef = ref
}
}

// AddNode provisions a worker node via bink, waits for it to be Ready,
// and returns the node name. The node is labeled with LabelE2ETest
// (and any extra labels from WithLabel).
Expand All @@ -122,6 +145,13 @@ func (e *Env) AddNode(t *testing.T, opts ...NodeOption) string {
o(cfg)
}

if cfg.targetImgRef == "" {
if e.nodeImageRegistry == "" || e.nodeImageDigest == "" {
t.Fatal("BINK_LOCAL_REGISTRY_NODE_IMAGE and NODE_IMAGE_DIGEST must be set (or use WithTargetImgRef)")
}
cfg.targetImgRef = e.nodeImageRegistry + "@" + e.nodeImageDigest
}

nodeName := e.generateNodeName(t)

// Provision the node with labels applied at join time.
Expand All @@ -133,9 +163,10 @@ func (e *Env) AddNode(t *testing.T, opts ...NodeOption) string {
if cfg.memory > 0 {
args = append(args, "--memory", fmt.Sprintf("%d", cfg.memory))
}
if img := os.Getenv("BINK_NODE_IMAGE"); img != "" {
if img := os.Getenv("BINK_NODE_DISK_IMAGE"); img != "" {
args = append(args, "--node-image", img)
}
args = append(args, "--target-imgref", cfg.targetImgRef)
t.Logf("Adding node %q...", nodeName)
if err := runBink(t, args...); err != nil {
t.Fatalf("adding node %q: %v", nodeName, err)
Expand Down Expand Up @@ -169,6 +200,20 @@ func (e *Env) TestLabels() map[string]string {
return map[string]string{LabelE2ETest: e.testID}
}

// NodeImageDigestedPullSpec returns the digest-qualified reference for the
// seeded node image (e.g. "registry.cluster.local:5000/node@sha256:abc123").
func (e *Env) NodeImageDigestedPullSpec() string {
if e.nodeImageRegistry == "" || e.nodeImageDigest == "" {
return ""
}
return e.nodeImageRegistry + "@" + e.nodeImageDigest
}

// NodeImageDigest returns the manifest digest of the seeded node image.
func (e *Env) NodeImageDigest() string {
return e.nodeImageDigest
}

// cleanup gathers diagnostic logs, then deletes test-scoped resources
// and bink nodes.
func (e *Env) cleanup(t *testing.T) {
Expand Down