Adds docker image example and instructions on how to create them

.github/workflows/_update_image.yml

@@ -0,0 +1,51 @@
+name: Update Docker Image
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+
+      - name: Generate Image Name
+        run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-image >> $GITHUB_ENV
+
+      - name: Log in to GitHub Docker Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker Metadata
+        id: meta
+        uses: docker/metadata-action@v6.1.0
+        with:
+          images: ${{ env.IMAGE_REPOSITORY }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=match,pattern=python-interface-to-workflows@v?(.+),group=1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4.0.0
+        with:
+          driver-opts: network=host
+
+      - name: Build Image
+        uses: docker/build-push-action@v6.18.0
+        with:
+          context: .
+          push: ${{ github.event_name == 'push' }}
+          load: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/ci.yml

@@ -17,6 +17,9 @@ jobs:
   commit-lint:
     uses: ./.github/workflows/_commit_msg.yml
 
+  updateimage:
+    uses: ./.github/workflows/_update_image.yml
+
   test:
     strategy:
       matrix:

.vscode/settings.json

@@ -9,5 +9,8 @@
     "[python]": {
         "editor.defaultFormatter": "charliermarsh.ruff",
     },
-    "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env"
+    "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env",
+    "python.testing.pytestArgs": [
+        "tests"
+    ]
 }

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -46,6 +47,7 @@ dev = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
 ]
 
 [project.scripts]

scripts/makecopiercorrect.sh

@@ -11,13 +11,15 @@ do
 
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
 
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
     mv "$file" "$file.jinja"
@@ -30,12 +32,14 @@ do
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
 
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
 
@@ -48,13 +52,14 @@ do
 
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
-
+    sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file"
     sed -i '1i{% raw %}' "$file"
     echo '{% endraw %}' >> "$file"
 
     sed -i \
         -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
         -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \
         "$file"
 
     mv "$file" "$file.jinja"

src/copier_template/pyproject.toml.jinja

@@ -24,6 +24,7 @@ dependencies = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -46,6 +47,7 @@ dev = [
     "h5py",
     "dotenv",
     "python-keycloak",
+    "pytest-asyncio",
 ]
 
 

src/copier_template/src/README.md.jinja

@@ -2,7 +2,7 @@
 1. run "uv lock" to generate the uv.lock file
 2. Create .env in this folder (with the path src/.env) containing the following variables:
 
-HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster)
+HOST=https://workflows.diamond.ac.uk/graphql (to submit to the production cluster)
 DEFAULT_IMAGE= (usually python 3.10)
 VISIT= (the Visit you wish to run the template on)
 TOKEN=
@@ -25,3 +25,20 @@ submit_workflow(w)
 
 NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are 
 by default, ran on pre-commit, to ensure that any yaml files they create are up to date.
+
+# Building a custom image
+While in src, the same folder as a Dockerfile:
+
+podman build -t ghcr.io/Your-Github-Name/image-name .
+podman login ghcr.io
+podman push ghcr.io/Your-Github-Name/image-name
+
+Then go to your github profile, packages, and set image-name's visibility to public
+After this, you may add 'image' in the script decorator, to run specific scripts within that image
+Alternatively, you can set the default image at the top of the file by adding:
+
+```python
+global_config.set_class_defaults(  # pyright: ignore
+    Script, image=str(os.environ.get("DEFAULT_IMAGE"))
+)
+```

src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja

@@ -9,7 +9,7 @@ from hera.workflows import Workflow
 from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable
 
 
-def submit_workflow(w: Workflow):
+async def submit_workflow(w: Workflow):
     yamlstr = w.to_yaml()  # pyright:ignore
     dotenv.load_dotenv(dotenv_path="src/.env", override=True)
     token: str = set_token_env_variable()
@@ -34,7 +34,7 @@ mutation Submit($visit: VisitInput!, $manifest: String!) {
   }
 }
 """)
-    result = client.execute(
+    result = await client.execute_async(
         mutation,
         variable_values={
             "visit": {

src/copier_template/src/{{ project_name }}/templates/example.txt.jinja

@@ -13,12 +13,12 @@ metadata:
     workflows.diamond.ac.uk/science-group-examples: 'true'
 spec:
   entrypoint: workflowentry
+  podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu":
+    "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}'
   templates:
   - name: workflowentry
     dag:
       tasks:
-      - name: install
-        template: install-dependencies
       - name: params
         template: generate-parameters
         arguments:
@@ -34,7 +34,7 @@ spec:
           - name: tiff
             value: 'True'
       - name: create-image
-        depends: install && params
+        depends: params
         template: create-image
         withParam: '{{tasks.params.outputs.parameters.out-parameters}}'
         arguments:
@@ -54,22 +54,6 @@ spec:
           parameters:
           - name: paths
             value: '{{tasks.create-image.outputs.parameters.out-paths}}'
-  - name: install-dependencies
-    script:
-      image: python:3.10
-      source: |-
-        import os
-        import sys
-        sys.path.append(os.getcwd())
-        import subprocess
-        print('creating venv')
-        subprocess.check_call(['python', '-m', 'venv', '/tmp/venv'])
-        subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera'])
-      command:
-      - python
-      volumeMounts:
-      - name: tmpdir
-        mountPath: /tmp
   - name: generate-parameters
     inputs:
       parameters:
@@ -84,7 +68,7 @@ spec:
         valueFrom:
           path: /tmp/parameters.json
     script:
-      image: python:3.10
+      image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image
       source: |-
         import os
         import sys
@@ -129,7 +113,7 @@ spec:
         valueFrom:
           path: /tmp/{{inputs.parameters.extension}}-path.json
     script:
-      image: python:3.10
+      image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image
       source: |-
         import os
         import sys
@@ -163,7 +147,7 @@ spec:
         with open(f'/tmp/{extension}-path.json', 'w') as f:
             json.dump(path, f)
       command:
-      - /tmp/venv/bin/python
+      - python
       volumeMounts:
       - name: tmpdir
         mountPath: /tmp
@@ -178,7 +162,7 @@ spec:
         archive:
           none: {}
     script:
-      image: python:3.10
+      image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image
       source: |-
         import os
         import sys
@@ -200,10 +184,19 @@ spec:
                     f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype)
         print('done')
       command:
-      - /tmp/venv/bin/python
+      - python
       volumeMounts:
       - name: tmpdir
         mountPath: /tmp
+  tolerations:
+  - effect: NoSchedule
+    key: nodetype
+    operator: Equal
+    value: gpu
+  - effect: NoSchedule
+    key: nodegroup
+    operator: Equal
+    value: workflows
   volumeClaimTemplates:
   - metadata:
       name: tmpdir