Chore: [AEA-0000] - setup common makefile (#18)

## Summary

- Routine Change

### Details

- setup common Makefile targets
- create seperate image for github actions with user vscode mapped to
user id 1001
- add cfn-guard and cfn-lint to image
- reduce size of docker images by using zstd compression

Author: anthony-nhs

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
 {
-    "name": "Ubuntu",
+    "name": "eps-devcontainers",
     // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
     "build": {
       "dockerfile": "Dockerfile",

.dockerignore

@@ -0,0 +1,4 @@
+.git
+node_modules
+.venv
+.out

.github/scripts/delete_unused_images.sh

@@ -44,14 +44,18 @@ delete_pr_images() {
 	fi
 
 	while IFS= read -r tag; do
-		if [[ "${tag}" =~ ^pr-[0-9]+- ]]; then
-			local pull_request
+		local pull_request
+		if [[ "${tag}" =~ ^pr-([0-9]+)- ]]; then
+			pull_request=${BASH_REMATCH[1]}
+		elif [[ "${tag}" =~ ^githubactions-pr-([0-9]+)$ ]]; then
+			pull_request=${BASH_REMATCH[1]}
+		else
+			continue
+		fi
+
 			local pr_json
 			local pr_state
 
-			pull_request=${tag#pr-}
-			pull_request=${pull_request%%-*}
-
 			if ! pr_json=$(gh api \
 				-H "Accept: application/vnd.github+json" \
 				"/repos/NHSDigital/eps-devcontainers/pulls/${pull_request}"); then
@@ -75,7 +79,6 @@ delete_pr_images() {
 						 	"/orgs/nhsdigital/packages/container/${package_name}/versions/${version_id}"
 					fi
 				done
-		fi
 	done <<<"${tags}"
 }
 

.github/workflows/build_multi_arch_image.yml

@@ -131,11 +131,17 @@ jobs:
         env:
           ARCHITECTURE: '${{ matrix.arch }}'
           DOCKER_TAG: '${{ inputs.docker_tag }}'
-      - name: Push tagged image
+      - name: Push tagged image and rebuild for github actions
         run: |
           echo "Pushing image..."
           docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}"
           echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "Rebuilding image for github actions with tag githubactions-${DOCKER_TAG}-${ARCHITECTURE}"
+          make build-githubactions-image BASE_IMAGE_NAME="${CONTAINER_NAME}" BASE_IMAGE_TAG="${DOCKER_TAG}-${ARCHITECTURE}" IMAGE_TAG="${DOCKER_TAG}-${ARCHITECTURE}" NO_CACHE="${{ inputs.NO_CACHE }}"
+          echo "Pushing github actions image..."
+          docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}"
+          echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY"
         env:
           DOCKER_TAG: ${{ inputs.docker_tag }}
           CONTAINER_NAME: '${{ inputs.container_name }}'
@@ -144,9 +150,14 @@ jobs:
         if: ${{ inputs.tag_latest }}
         run: |
           docker tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-${ARCHITECTURE}" "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}"
-          echo "Pushing image..."
+          echo "Pushing latest image..."
           docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}"
           echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY"
+
+          docker tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-${ARCHITECTURE}" "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}"
+          echo "Pushing github actions latest image..."
+          docker push "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}"
+          echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-${ARCHITECTURE}" >> "$GITHUB_STEP_SUMMARY"
         env:
           DOCKER_TAG: ${{ inputs.docker_tag }}
           CONTAINER_NAME: '${{ inputs.container_name }}'
@@ -172,6 +183,7 @@ jobs:
         run: |
           BUILD_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
           export BUILD_TIMESTAMP
+          echo "Creating combined image for tag ${DOCKER_TAG}"
           docker buildx imagetools create \
             --annotation "index:org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers" \
             --annotation "index:org.opencontainers.image.description=EPS devcontainer ${CONTAINER_NAME}:${DOCKER_TAG}" \
@@ -184,17 +196,39 @@ jobs:
             "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-amd64" \
             "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}-arm64"
           echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:${DOCKER_TAG}" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "Creating combined image for tag githubactions-${DOCKER_TAG}"
+          docker buildx imagetools create \
+            --annotation "index:org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers" \
+            --annotation "index:org.opencontainers.image.description=EPS devcontainer ${CONTAINER_NAME}:${DOCKER_TAG}" \
+            --annotation "index:org.opencontainers.image.licenses=MIT" \
+            --annotation "index:org.opencontainers.image.version=${DOCKER_TAG}" \
+            --annotation "index:org.opencontainers.image.containerName=${CONTAINER_NAME}" \
+            --annotation "index:org.opencontainers.image.created=${BUILD_TIMESTAMP}" \
+            --annotation "index:org.opencontainers.image.authors=NHS England EPS Team" \
+            --tag "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}" \
+            "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-amd64" \
+            "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}-arm64"
+          echo "## PUSHED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-${DOCKER_TAG}" >> "$GITHUB_STEP_SUMMARY"
         env:
           DOCKER_TAG: ${{ inputs.docker_tag }}
           CONTAINER_NAME: '${{ inputs.container_name }}'
 
       - name: Push multi-arch latest image
         if: ${{ inputs.tag_latest }}
         run: |
+          echo "Creating combined image for tag latest"
           docker buildx imagetools create -t "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest" \
             "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-amd64" \
             "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest-arm64"
           echo "## PUSHED COMBINED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:latest" >> "$GITHUB_STEP_SUMMARY"
+
+          echo "Creating combined image for tag githubactions-latest"
+          docker buildx imagetools create -t "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest" \
+            "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-amd64" \
+            "ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest-arm64"
+          echo "## PUSHED COMBINED IMAGE : ghcr.io/nhsdigital/eps-devcontainers/${CONTAINER_NAME}:githubactions-latest" >> "$GITHUB_STEP_SUMMARY"
+
         env:
           DOCKER_TAG: ${{ inputs.docker_tag }}
           CONTAINER_NAME: '${{ inputs.container_name }}'

Makefile

@@ -1,9 +1,5 @@
 CONTAINER_PREFIX=ghcr.io/nhsdigital/eps-devcontainers/
 
-ifneq ($(strip $(PLATFORM)),)
-PLATFORM_FLAG=--platform $(PLATFORM)
-endif
-
 ifeq ($(strip $(NO_CACHE)),true)
 NO_CACHE_FLAG=--no-cache
 endif
@@ -30,9 +26,20 @@ build-image: guard-CONTAINER_NAME guard-BASE_VERSION_TAG guard-BASE_FOLDER guard
 		--workspace-folder ./src/$${BASE_FOLDER}/$${CONTAINER_NAME} \
 		$(NO_CACHE_FLAG) \
 		--push false \
+		--output type=image,name="${CONTAINER_PREFIX}$${CONTAINER_NAME}:$${IMAGE_TAG}",push=false,compression=zstd \
 		--cache-from "${CONTAINER_PREFIX}$${CONTAINER_NAME}:latest" \
 		--image-name "${CONTAINER_PREFIX}$${CONTAINER_NAME}:$${IMAGE_TAG}"
 
+build-githubactions-image: guard-BASE_IMAGE_NAME guard-BASE_IMAGE_TAG guard-IMAGE_TAG
+	docker buildx build \
+		-f src/githubactions/Dockerfile \
+		$(NO_CACHE_FLAG) \
+		--build-arg BASE_IMAGE_NAME="$${BASE_IMAGE_NAME}" \
+		--build-arg BASE_IMAGE_TAG="$${BASE_IMAGE_TAG}" \
+		--load \
+		-t "${CONTAINER_PREFIX}$${BASE_IMAGE_NAME}:githubactions-$${IMAGE_TAG}" \
+		.
+
 scan-image: guard-CONTAINER_NAME guard-BASE_FOLDER
 	@combined="src/$${BASE_FOLDER}/$${CONTAINER_NAME}/.trivyignore_combined.yaml"; \
 	common="src/common/.trivyignore.yaml"; \

README.md

@@ -8,8 +8,6 @@ Images are built using using https://github.com/devcontainers/cli.
 
 We build a base image based on mcr.microsoft.com/devcontainers/base:ubuntu-22.04 that other images are then based on
 
-The images have vsocde user setup as user 1001 so that they can be used in github actions
-
 The base image contains
  - latest os packages
  - asdf
@@ -109,11 +107,12 @@ This job should be used in github actions wherever you need to get the dev conta
           echo "DEVCONTAINER_IMAGE_VERSION=$DEVCONTAINER_VERSION" >> "$GITHUB_OUTPUT"
 ```
 # Project structure
-We have 3 types of dev container. These are defined under src
+We have 4 types of dev container. These are defined under src
 
 `base` - this is the base image that all others are based on.   
 `languages` - this installs specific versions of node and python.   
-`projects` - this is used for projects where more customization is needed than just a base language image
+`projects` - this is used for projects where more customization is needed than just a base language image.   
+`githubactions` - this just takes an existing image and remaps vscode user to be 1001 so it can be used by github actions.   
 
 Each image to be built contains a .devcontainer folder that defines how the devcontainer should be built. At a minimum, this should contain a devcontainer.json file. See https://containers.dev/implementors/json_reference/ for options for this
 
@@ -122,16 +121,20 @@ Images under languages should point to a dockerfile under src/common that is bas
 We use trivy to scan for vulnerabilities in the built docker images. Known vulnerabilities in the base image are in `src/common/.trivyignore.yaml`. Vulnerabilities in specific images are in `.trivyignore.yaml` file in each images folder. These are combined before running a scan to exclude all known vulnerabilities
 
 # Pull requests and merge to main process
-For each pull request, and merge to main, images are built and scanned using trivy, but the images are not pushed to github container registry
-Docker images are built for each pull request, and on merges to main.   
-Docker images are built for amd64 and arm64 architecture, and a combined manifest is created and pushed as part of the build.   
+For each pull request, and merge to main, images are built and scanned using trivy, and pushed to github docker registry.      
+Docker images are built for amd64 and arm64 architecture, and a combined manifest is created and pushed as part of the build.
+The main images have a vscode user with id 1000. A separately tagged image is also created with user vscode mapped to user id 1001 so they can be used by github actions.
 
 The base image is built first, and then language images, and finally project images. 
 
 Docker images are scanned for vulnerabilities using trivy as part of a build step, and the build fails if vulnerabilities are found not in .trivyignore file.
 
-For pull requests, images are tagged with the pr-<pull request id>-<short commit sha>.   
-For merges to main, images are tagged with the <short commit sha>.   
+For pull requests, images are tagged with the pr-{pull request id}-{short commit sha}.   
+For merges to main, images are tagged with the {short commit sha}.   
+Github actions images are tagged with githubactions-{tag}
+Amd64 images are tagged with {tag}-amd64
+Arm64 images are tagged with {tag}-arm64
+Combined image manifest image is just tagged with {tag} so can be included in devcontainer.json and the correct image is pulled based on the host architecture.   
 
 When a pull request is merged to main or closed, all associated images are deleted from the registry using the github workflow delete_old_images
 
@@ -154,7 +157,7 @@ CONTAINER_NAME=base \
 ``` 
 Language images
 ```
-CONTAINER_NAME=node_24_python_3_13 \
+CONTAINER_NAME=node_24_python_3_14 \
   BASE_VERSION_TAG=local-build \
   BASE_FOLDER=languages \
   IMAGE_TAG=local-build \
@@ -168,7 +171,13 @@ CONTAINER_NAME=fhir_facade_api \
   IMAGE_TAG=local-build \
   make build-image
 ``` 
-
+Github actions image
+```
+BASE_IMAGE_NAME=base \
+  BASE_IMAGE_TAG=local-build \
+  IMAGE_TAG=local-build \
+  make build-githubactions-image
+```
 ## Scanning images
 You can use these commands to scan images
 Base image
@@ -213,13 +222,39 @@ CONTAINER_NAME=fhir_facade_api \
   IMAGE_TAG=local-build \
   make shell-image
 ``` 
+github actions image
+```
+CONTAINER_NAME=base \
+  IMAGE_TAG=githubactions-local-build \
+  make shell-image
+```
 
-## Using local or pull request images
+## Using local or pull request images in visual studio code
 You can use local or pull request images by changing IMAGE_VERSION in devcontainer.json.    
 For an image built locally, you should put the IMAGE_VERSION=local-build. 
 For an image built from a pull request, you should put the IMAGE_VERSION=<tag of image as show in pull request job>.  
 You can only use images built from a pull request for testing changes in github actions.   
 
+## Using images in github actions
+To use the image in github actions, you can use it in github actions using code like this
+```
+jobs:
+  my_job_name:
+    runs-on: ubuntu-22.04
+    container:
+      image: ghcr.io/nhsdigital/eps-devcontainers/<container name>:githubactions-<tag>
+      options: --user 1001:1001
+    steps:
+      - name: copy .tool-versions
+        run: |
+          cp /home/vscode/.tool-versions "$HOME/.tool-versions"
+      ... other steps ....
+```
+It is important that 
+- the image uses the tag starting githubactions-
+- there is `options: --user 1001:1001` below image
+- the first step copies .tool-versions from /home/vscode to $HOME/.tool-versions
+
 ## Generating a .trivyignore file
 You can generate a .trivyignore file for known vulnerabilities by either downloading the json scan output generated by the build, or by generating it locally using the scanning images commands above with a make target of scan-image-json
 
@@ -238,3 +273,49 @@ poetry run python \
   --input .out/scan_results_docker.json \
   --output src/projects/fhir_facade_api/.trivyignore.new.yaml 
 ```
+
+## Common makefile targets
+There are a set of common Makefiles that are defined in `src/base/.devcontainer/Mk` and are included from `common.mk`. These are installed to /usr/local/share/eps/Mk on the base image so are available for all containers.
+
+This should be added to the end of each projects Makefile to include them
+```
+%:
+	@$(MAKE) -f /usr/local/share/eps/Mk/common.mk $@
+```
+### Targets
+The following targets are defined. These are needed for quality checks to run. Some targets are project specific and so should be overridden in the projects Makefile.
+
+Build targets (`build.mk`)
+- `install` - placeholder target - should be overridden locally
+- `install-node` - placeholder target - should be overridden locally
+- `docker-build` - placeholder target - should be overridden locally
+- `compile` - placeholder target - should be overridden locally
+
+Check targets (`check.mk`)
+- `lint` - placeholder target - should be overridden locally
+- `test` - placeholder target - should be overridden locally
+- `shellcheck` - runs shellcheck on `scripts/*.sh` and `.github/scripts/*.sh` when files exist
+- `cfn-lint` - runs `cfn-lint` against `cloudformation/**/*.yml|yaml` and `SAMtemplates/**/*.yml|yaml`
+- `cdk-synth` - placeholder target - should be overridden locally
+- `cfn-guard-sam-templates` - validates SAM templates against cfn-guard rulesets and writes outputs to `.cfn_guard_out/`
+- `cfn-guard-cloudformation` - validates `cloudformation` templates against cfn-guard rulesets and writes outputs to `.cfn_guard_out/`
+- `cfn-guard-cdk` - validates `cdk.out` against cfn-guard rulesets and writes outputs to `.cfn_guard_out/`
+- `cfn-guard-terraform` - validates `terraform_plans` against cfn-guard rulesets and writes outputs to `.cfn_guard_out/`
+- `actionlint` - runs actionlint against github actions
+- `secret-scan` - runs git-secrets (including scanning history) against the repo
+- `guard-<ENVIRONMENT_VARIABLE>` - checks if an environment variable is set and errors if it is not
+
+Credentials targets (`credentials.mk`)
+- `aws-configure` - configures an AWS sso session
+- `aws-login` - Authorizes an sso session with AWS so aws cli tools can be used. You may still need to set AWS_PROFILE before running commands
+- `github-login` - Authorizes github cli to github with scope to read packages
+- `create-npmrc` - depends on `github-login`, then writes `.npmrc` with a GitHub Packages auth token and `@nhsdigital` registry
+
+Trivy targets (`trivy.mk`)
+- `trivy-license-check` - runs Trivy license scan (HIGH/CRITICAL) and writes `.trivy_out/license_scan.txt`
+- `trivy-generate-sbom` - generates CycloneDX SBOM at `.trivy_out/sbom.cdx.json`
+- `trivy-scan-python` - scans Python dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_python.txt`
+- `trivy-scan-node` - scans Node dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_node.txt`
+- `trivy-scan-go` - scans Go dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_go.txt`
+- `trivy-scan-java` - scans Java dependencies (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_java.txt`
+- `trivy-scan-docker` - scans a built image (HIGH/CRITICAL) and writes `.trivy_out/dependency_results_docker.txt` (requires `DOCKER_IMAGE`), for example:

src/base/.devcontainer/Dockerfile

@@ -2,39 +2,37 @@ FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04
 
 ARG SCRIPTS_DIR=/usr/local/share/eps
 ARG CONTAINER_NAME
-ARG MULTI_ARCH_TAG
-ARG BASE_VERSION_TAG
 ARG IMAGE_TAG
 ARG TARGETARCH
 
 ENV SCRIPTS_DIR=${SCRIPTS_DIR}
 ENV CONTAINER_NAME=${CONTAINER_NAME}
-ENV MULTI_ARCH_TAG=${MULTI_ARCH_TAG}
-ENV BASE_VERSION_TAG=${BASE_VERSION_TAG}
-ENV IMAGE_TAG=${IMAGE_TAG}
 ENV TARGETARCH=${TARGETARCH}
 
-LABEL org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers
-LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}"
-LABEL org.opencontainers.image.licenses=MIT
-LABEL org.opencontainers.image.version=${IMAGE_TAG}
-LABEL org.opencontainers.image.containerName=${CONTAINER_NAME}
-LABEL org.opencontainers.image.authors="NHS England EPS Team"
-LABEL org.opencontainers.image.base.image="mcr.microsoft.com/devcontainers/base:ubuntu-22.04"
-
 COPY .tool-versions.asdf ${SCRIPTS_DIR}/${CONTAINER_NAME}/.tool-versions.asdf
-COPY --chmod=755 scripts ${SCRIPTS_DIR}/${CONTAINER_NAME}
+COPY --chmod=755 scripts/root_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/root_install.sh
+COPY --chmod=755 Mk ${SCRIPTS_DIR}/Mk
 
 WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME}
 RUN ./root_install.sh
 
+COPY --chmod=755 scripts/vscode_install.sh ${SCRIPTS_DIR}/${CONTAINER_NAME}/vscode_install.sh
 USER vscode
 COPY --chown=vscode:vscode .tool-versions.asdf /home/vscode/.tool-versions.asdf
 COPY --chown=vscode:vscode .tool-versions /home/vscode/.tool-versions
 
-ENV PATH="/home/vscode/.asdf/shims/:$PATH"
+ENV PATH="/home/vscode/.asdf/shims/:/home/vscode/.guard/bin/:$PATH"
 WORKDIR ${SCRIPTS_DIR}/${CONTAINER_NAME}
 RUN ./vscode_install.sh
 
 # Switch back to root to install the devcontainer CLI globally
 USER root
+
+ENV IMAGE_TAG=${IMAGE_TAG}
+
+LABEL org.opencontainers.image.source=https://github.com/NHSDigital/eps-devcontainers
+LABEL org.opencontainers.image.description="EPS devcontainer ${CONTAINER_NAME}:${IMAGE_TAG}"
+LABEL org.opencontainers.image.licenses=MIT
+LABEL org.opencontainers.image.containerName=${CONTAINER_NAME}
+LABEL org.opencontainers.image.authors="NHS England EPS Team"
+LABEL org.opencontainers.image.base.image="mcr.microsoft.com/devcontainers/base:ubuntu-22.04"

src/base/.devcontainer/Mk/build.mk

@@ -0,0 +1,16 @@
+.PHONY: install install-node docker-build compile
+install:
+	echo "Not implemented"
+	exit 1
+
+install-node:
+	echo "Not implemented"
+	exit 1
+
+docker-build:
+	echo "Not implemented"
+	exit 1
+
+compile:
+	echo "Not implemented"
+	exit 1