From 8b0d2264f32750caae19d38e34ebb45317d4ad71 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 24 Jul 2026 15:09:48 +0000 Subject: [PATCH 1/6] feat(example): adds an additional example with default image, updates copier --- Dockerfile | 4 +- .../templates/example_in_image.txt.jinja | 196 ++++++++++++++++++ ...ple_template_within_default_image.py.jinja | 172 +++++++++++++++ .../templates/example_in_image.yaml | 196 ++++++++++++++++++ ...e_example_template_within_default_image.py | 172 +++++++++++++++ 5 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja create mode 100644 src/python_interface_to_workflows/templates/example_in_image.yaml create mode 100644 src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py diff --git a/Dockerfile b/Dockerfile index 0736493..bfc7efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python # Sync the project without its dev dependencies RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable --no-dev --managed-python + uv sync --locked --no-dev --no-editable --managed-python # The runtime stage copies the built venv into a runtime container FROM ubuntu:resolute AS runtime @@ -43,7 +43,7 @@ FROM ubuntu:resolute AS runtime # Copy the python installation from the build stage COPY --from=build /python /python -# Copy the environment, but not the source code +# Copy the environment, and the source code COPY --from=build /app/.venv /app/.venv ENV PATH=/app/.venv/bin:$PATH diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja new file mode 100644 index 0000000..6614a15 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -0,0 +1,196 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hera-example-in-image- + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + example.yaml + workflows.argoproj.io/title: example remade via hera + workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: params + template: generate-parameters + arguments: + parameters: + - name: png + value: 'True' + - name: jpg + value: 'True' + - name: jpeg + value: 'True' + - name: tif + value: 'True' + - name: tiff + value: 'True' + - name: create-image + depends: params + template: create-image + withParam: '{{tasks.params.outputs.parameters.out-parameters}}' + arguments: + parameters: + - name: width + value: '{{item.width}}' + - name: height + value: '{{item.height}}' + - name: weights + value: '{{item.weights}}' + - name: extension + value: '{{item.extension}}' + - name: to-hdf5 + depends: create-image + template: to-hdf5 + arguments: + parameters: + - name: paths + value: '{{tasks.create-image.outputs.parameters.out-paths}}' + - name: generate-parameters + inputs: + parameters: + - name: png + - name: jpg + - name: jpeg + - name: tif + - name: tiff + outputs: + parameters: + - name: out-parameters + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') + except: jpeg = r'''{{inputs.parameters.jpeg}}''' + try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') + except: jpg = r'''{{inputs.parameters.jpg}}''' + try: png = json.loads(r'''{{inputs.parameters.png}}''') + except: png = r'''{{inputs.parameters.png}}''' + try: tif = json.loads(r'''{{inputs.parameters.tif}}''') + except: tif = r'''{{inputs.parameters.tif}}''' + try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') + except: tiff = r'''{{inputs.parameters.tiff}}''' + + import json + params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] + params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] + with open('/tmp/parameters.json', 'w') as f: + json.dump(params_to_write, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: create-image + inputs: + parameters: + - name: width + - name: height + - name: weights + - name: extension + outputs: + artifacts: + - name: '{{inputs.parameters.extension}}-image' + path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} + archive: + none: {} + parameters: + - name: out-paths + valueFrom: + path: /tmp/{{inputs.parameters.extension}}-path.json + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: extension = json.loads(r'''{{inputs.parameters.extension}}''') + except: extension = r'''{{inputs.parameters.extension}}''' + try: height = json.loads(r'''{{inputs.parameters.height}}''') + except: height = r'''{{inputs.parameters.height}}''' + try: weights = json.loads(r'''{{inputs.parameters.weights}}''') + except: weights = r'''{{inputs.parameters.weights}}''' + try: width = json.loads(r'''{{inputs.parameters.width}}''') + except: width = r'''{{inputs.parameters.width}}''' + + import json + from PIL import Image + + def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: + print(f'width: {width}') + print(f'height: {height}') + print(f'RBG weights: {weights}') + image = Image.new('RGB', (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) + return image + image = create_pattern(width, height, weights) + path = f'/tmp/{extension}-image.{extension}' + image.save(path) + with open(f'/tmp/{extension}-path.json', 'w') as f: + json.dump(path, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: to-hdf5 + inputs: + parameters: + - name: paths + outputs: + artifacts: + - name: hdf5output + path: /tmp/images.hdf5 + archive: + none: {} + script: + image: ghcr.io/matt-carre/{{repo_name}}-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: paths = json.loads(r'''{{inputs.parameters.paths}}''') + except: paths = r'''{{inputs.parameters.paths}}''' + + import h5py + import numpy as np + from PIL import Image + print('creating hdf5 file') + with h5py.File('/tmp/images.hdf5', 'w') as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f'Got {path}') + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) + print('done') + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja new file mode 100644 index 0000000..eaaf2d4 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -0,0 +1,172 @@ +import os + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Parameter, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) + + +@script( + command=["python"], + outputs=Parameter( + name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def generate_parameters( + png: str, + jpg: str, + jpeg: str, + tif: str, + tiff: str, +): + import json + + params: list[dict[str, int | list[int] | str] | None] = [ + {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} + if png.lower() == "true" + else None, + {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} + if jpg.lower() == "true" + else None, + {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} + if jpeg.lower() == "true" + else None, + {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} + if tif.lower() == "true" + else None, + {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} + if tiff.lower() == "true" + else None, + ] + params_to_write: list[dict[str, int | list[int] | str]] = [ + image_params for image_params in params if image_params is not None + ] + with open("/tmp/parameters.json", "w") as f: + json.dump(params_to_write, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=[ + Parameter( + name="out-paths", + value_from=m.ValueFrom( + path="/tmp/{{inputs.parameters.extension}}-path.json" + ), + ), + Artifact( + name="{{inputs.parameters.extension}}-image", + path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", + archive=NoneArchiveStrategy(), + ), + ], +) +def create_image( + width: int, height: int, weights: tuple[int, int, int], extension: str +): + import json + + from PIL import Image + + def create_pattern( + width: int, + height: int, + weights: tuple[int, int, int], + ) -> Image.Image: + print(f"width: {width}") + print(f"height: {height}") + print(f"RBG weights: {weights}") + image = Image.new("RGB", (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] + (i + j * 50) % weights[0], + weights[1], + (i * 300 + j) % weights[2], + ) + return image + + image = create_pattern(width, height, weights) + path = f"/tmp/{extension}-image.{extension}" + image.save(path) + with open(f"/tmp/{extension}-path.json", "w") as f: + json.dump(path, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=Artifact( + name="hdf5output", + path="/tmp/images.hdf5", + archive=NoneArchiveStrategy(), + ), +) +def to_hdf5(paths: str): + + import h5py # pyright: ignore[reportMissingTypeStubs] + import numpy as np + from PIL import Image + + print("creating hdf5 file") + with h5py.File("/tmp/images.hdf5", "w") as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f"Got {path}") + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset( # pyright: ignore[reportUnknownMemberType] + f"image_{i}", data=arr, dtype=arr.dtype + ) + print("done") + + +with Workflow( + generate_name="hera-example-in-image-", # name on graphql + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="Workflow", # ClusterWorkflowTemplate", when on graphql + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "example remade via hera", + "workflows.argoproj.io/description": """Replicates the functionality of +example.yaml""", + "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + params = generate_parameters( + name="params", + arguments={ + "png": "True", + "jpg": "True", + "jpeg": "True", + "tif": "True", + "tiff": "True", + }, + ) + makeimages = create_image(with_param=params.get_parameter("out-parameters")) + makehdf5 = to_hdf5( + arguments={ + "paths": makeimages.get_parameter("out-paths"), + } + ) + params >> makeimages >> makehdf5 # pyright: ignore +with open("example_in_image.txt", "w") as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/templates/example_in_image.yaml b/src/python_interface_to_workflows/templates/example_in_image.yaml new file mode 100644 index 0000000..64f9565 --- /dev/null +++ b/src/python_interface_to_workflows/templates/example_in_image.yaml @@ -0,0 +1,196 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hera-example-in-image- + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + example.yaml + workflows.argoproj.io/title: example remade via hera + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: params + template: generate-parameters + arguments: + parameters: + - name: png + value: 'True' + - name: jpg + value: 'True' + - name: jpeg + value: 'True' + - name: tif + value: 'True' + - name: tiff + value: 'True' + - name: create-image + depends: params + template: create-image + withParam: '{{tasks.params.outputs.parameters.out-parameters}}' + arguments: + parameters: + - name: width + value: '{{item.width}}' + - name: height + value: '{{item.height}}' + - name: weights + value: '{{item.weights}}' + - name: extension + value: '{{item.extension}}' + - name: to-hdf5 + depends: create-image + template: to-hdf5 + arguments: + parameters: + - name: paths + value: '{{tasks.create-image.outputs.parameters.out-paths}}' + - name: generate-parameters + inputs: + parameters: + - name: png + - name: jpg + - name: jpeg + - name: tif + - name: tiff + outputs: + parameters: + - name: out-parameters + valueFrom: + path: /tmp/parameters.json + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') + except: jpeg = r'''{{inputs.parameters.jpeg}}''' + try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') + except: jpg = r'''{{inputs.parameters.jpg}}''' + try: png = json.loads(r'''{{inputs.parameters.png}}''') + except: png = r'''{{inputs.parameters.png}}''' + try: tif = json.loads(r'''{{inputs.parameters.tif}}''') + except: tif = r'''{{inputs.parameters.tif}}''' + try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') + except: tiff = r'''{{inputs.parameters.tiff}}''' + + import json + params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] + params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] + with open('/tmp/parameters.json', 'w') as f: + json.dump(params_to_write, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: create-image + inputs: + parameters: + - name: width + - name: height + - name: weights + - name: extension + outputs: + artifacts: + - name: '{{inputs.parameters.extension}}-image' + path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} + archive: + none: {} + parameters: + - name: out-paths + valueFrom: + path: /tmp/{{inputs.parameters.extension}}-path.json + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: extension = json.loads(r'''{{inputs.parameters.extension}}''') + except: extension = r'''{{inputs.parameters.extension}}''' + try: height = json.loads(r'''{{inputs.parameters.height}}''') + except: height = r'''{{inputs.parameters.height}}''' + try: weights = json.loads(r'''{{inputs.parameters.weights}}''') + except: weights = r'''{{inputs.parameters.weights}}''' + try: width = json.loads(r'''{{inputs.parameters.width}}''') + except: width = r'''{{inputs.parameters.width}}''' + + import json + from PIL import Image + + def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: + print(f'width: {width}') + print(f'height: {height}') + print(f'RBG weights: {weights}') + image = Image.new('RGB', (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) + return image + image = create_pattern(width, height, weights) + path = f'/tmp/{extension}-image.{extension}' + image.save(path) + with open(f'/tmp/{extension}-path.json', 'w') as f: + json.dump(path, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: to-hdf5 + inputs: + parameters: + - name: paths + outputs: + artifacts: + - name: hdf5output + path: /tmp/images.hdf5 + archive: + none: {} + script: + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: paths = json.loads(r'''{{inputs.parameters.paths}}''') + except: paths = r'''{{inputs.parameters.paths}}''' + + import h5py + import numpy as np + from PIL import Image + print('creating hdf5 file') + with h5py.File('/tmp/images.hdf5', 'w') as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f'Got {path}') + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) + print('done') + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py new file mode 100644 index 0000000..88740ca --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py @@ -0,0 +1,172 @@ +import os + +from hera.shared import global_config +from hera.workflows import ( + DAG, + Artifact, + Parameter, + Script, + Volume, + Workflow, + script, # pyright: ignore[reportUnknownVariableType] +) +from hera.workflows import models as m +from hera.workflows.archive import NoneArchiveStrategy + +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) + + +@script( + command=["python"], + outputs=Parameter( + name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") + ), + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +) +def generate_parameters( + png: str, + jpg: str, + jpeg: str, + tif: str, + tiff: str, +): + import json + + params: list[dict[str, int | list[int] | str] | None] = [ + {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} + if png.lower() == "true" + else None, + {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} + if jpg.lower() == "true" + else None, + {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} + if jpeg.lower() == "true" + else None, + {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} + if tif.lower() == "true" + else None, + {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} + if tiff.lower() == "true" + else None, + ] + params_to_write: list[dict[str, int | list[int] | str]] = [ + image_params for image_params in params if image_params is not None + ] + with open("/tmp/parameters.json", "w") as f: + json.dump(params_to_write, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=[ + Parameter( + name="out-paths", + value_from=m.ValueFrom( + path="/tmp/{{inputs.parameters.extension}}-path.json" + ), + ), + Artifact( + name="{{inputs.parameters.extension}}-image", + path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", + archive=NoneArchiveStrategy(), + ), + ], +) +def create_image( + width: int, height: int, weights: tuple[int, int, int], extension: str +): + import json + + from PIL import Image + + def create_pattern( + width: int, + height: int, + weights: tuple[int, int, int], + ) -> Image.Image: + print(f"width: {width}") + print(f"height: {height}") + print(f"RBG weights: {weights}") + image = Image.new("RGB", (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] + (i + j * 50) % weights[0], + weights[1], + (i * 300 + j) % weights[2], + ) + return image + + image = create_pattern(width, height, weights) + path = f"/tmp/{extension}-image.{extension}" + image.save(path) + with open(f"/tmp/{extension}-path.json", "w") as f: + json.dump(path, f) + + +@script( + command=["python"], + volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], + outputs=Artifact( + name="hdf5output", + path="/tmp/images.hdf5", + archive=NoneArchiveStrategy(), + ), +) +def to_hdf5(paths: str): + + import h5py # pyright: ignore[reportMissingTypeStubs] + import numpy as np + from PIL import Image + + print("creating hdf5 file") + with h5py.File("/tmp/images.hdf5", "w") as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f"Got {path}") + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset( # pyright: ignore[reportUnknownMemberType] + f"image_{i}", data=arr, dtype=arr.dtype + ) + print("done") + + +with Workflow( + generate_name="hera-example-in-image-", # name on graphql + entrypoint="workflowentry", + api_version="argoproj.io/v1alpha1", + kind="Workflow", # ClusterWorkflowTemplate", when on graphql + labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, + annotations={ + "workflows.argoproj.io/title": "example remade via hera", + "workflows.argoproj.io/description": """Replicates the functionality of +example.yaml""", + "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", + }, + volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), +) as w: + with DAG(name="workflowentry"): + params = generate_parameters( + name="params", + arguments={ + "png": "True", + "jpg": "True", + "jpeg": "True", + "tif": "True", + "tiff": "True", + }, + ) + makeimages = create_image(with_param=params.get_parameter("out-parameters")) + makehdf5 = to_hdf5( + arguments={ + "paths": makeimages.get_parameter("out-paths"), + } + ) + params >> makeimages >> makehdf5 # pyright: ignore +with open("example_in_image.txt", "w") as div: + div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] From 1fbf5e93f133eba126c760adcae179fa4b18065e Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 27 Jul 2026 09:12:39 +0000 Subject: [PATCH 2/6] docs(copier): adds instructions on how to create and set images in copier readme --- Dockerfile | 2 +- src/copier_template/src/README.md.jinja | 17 +++++++++++++++++ .../templates/example_in_image.txt.jinja | 10 ++++++---- ...ample_template_within_default_image.py.jinja | 4 +++- ...ample_in_image.yaml => example_in_image.txt} | 0 5 files changed, 27 insertions(+), 6 deletions(-) rename src/python_interface_to_workflows/templates/{example_in_image.yaml => example_in_image.txt} (100%) diff --git a/Dockerfile b/Dockerfile index bfc7efa..2a69ffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ FROM ubuntu:resolute AS runtime # Copy the python installation from the build stage COPY --from=build /python /python -# Copy the environment, and the source code +# Copy the environment, but not the source code COPY --from=build /app/.venv /app/.venv ENV PATH=/app/.venv/bin:$PATH diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index c09fef6..828529c 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -25,3 +25,20 @@ submit_workflow(w) NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are by default, ran on pre-commit, to ensure that any yaml files they create are up to date. + +# Building a custom image +While in src, the same folder as a Dockerfile: + +podman build -t ghcr.io/Your-Github-Name/image-name . +podman login ghcr.io +podman push ghcr.io/Your-Github-Name/image-name + +Then go to your github profile, packages, and set image-name's visibility to public +After this, you may add 'image' in the script decorator, to run specific scripts within that image +Alternatively, you can set the default image at the top of the file by adding: + +```python +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) +) +``` diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja index 6614a15..78b1316 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -1,3 +1,4 @@ +{% raw %} apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: @@ -7,7 +8,7 @@ metadata: Replicates the functionality of example.yaml workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} labels: workflows.diamond.ac.uk/science-group-examples: 'true' spec: @@ -65,7 +66,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -110,7 +111,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -159,7 +160,7 @@ spec: archive: none: {} script: - image: ghcr.io/matt-carre/{{repo_name}}-default-image + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -194,3 +195,4 @@ spec: resources: requests: storage: 1Gi +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja index eaaf2d4..80e5e29 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -1,3 +1,4 @@ +{% raw %} import os from hera.shared import global_config @@ -146,7 +147,7 @@ with Workflow( "workflows.argoproj.io/title": "example remade via hera", "workflows.argoproj.io/description": """Replicates the functionality of example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", + "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: @@ -170,3 +171,4 @@ example.yaml""", params >> makeimages >> makehdf5 # pyright: ignore with open("example_in_image.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] +{% endraw %} diff --git a/src/python_interface_to_workflows/templates/example_in_image.yaml b/src/python_interface_to_workflows/templates/example_in_image.txt similarity index 100% rename from src/python_interface_to_workflows/templates/example_in_image.yaml rename to src/python_interface_to_workflows/templates/example_in_image.txt From 6efc388b939a85c4f23a99bb7150b10d5dbf8c2b Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 11:09:50 +0000 Subject: [PATCH 3/6] feat(example): updates templates and copier files to new standards --- .../{{ project_name }}/templates/example_in_image.txt.jinja | 4 ++-- .../create_example_template_within_default_image.py.jinja | 4 ++-- .../notebooks/notebook_example.ipynb.jinja | 4 ++-- .../templates/example_in_image.txt | 4 ++-- .../create_example_template_within_default_image.py | 4 ++-- .../workflow_definitions/notebooks/notebook_division.ipynb | 6 +++--- .../workflow_definitions/notebooks/notebook_example.ipynb | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja index 78b1316..2606a5f 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja @@ -1,8 +1,8 @@ {% raw %} apiVersion: argoproj.io/v1alpha1 -kind: Workflow +kind: WorkflowTemplate metadata: - generateName: hera-example-in-image- + name: hera-example-in-image annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja index 80e5e29..3e604a5 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja @@ -138,10 +138,10 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-in-image-", # name on graphql + name="hera-example-in-image", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="Workflow", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja index 0f829f6..41e2409 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -166,10 +166,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " name=\"hera-example-\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", diff --git a/src/python_interface_to_workflows/templates/example_in_image.txt b/src/python_interface_to_workflows/templates/example_in_image.txt index 64f9565..ac6be98 100644 --- a/src/python_interface_to_workflows/templates/example_in_image.txt +++ b/src/python_interface_to_workflows/templates/example_in_image.txt @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 -kind: Workflow +kind: WorkflowTemplate metadata: - generateName: hera-example-in-image- + name: hera-example-in-image annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py index 88740ca..779921c 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py @@ -137,10 +137,10 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-in-image-", # name on graphql + name="hera-example-in-image", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="Workflow", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 9113672..107824e 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "494174ef", "metadata": {}, "outputs": [], @@ -50,10 +50,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " name=\"hera-division\",\n", " entrypoint=\"divide\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb index c1018ea..4d3b47f 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -166,10 +166,10 @@ "\n", "\n", "with Workflow(\n", - " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " kind=\"WorkflowTemplate\",\n", " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", " annotations={\n", " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", From 2ded032e9288d92f60ff4a9fc8ce9900c0bfcbaa Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 11:39:19 +0000 Subject: [PATCH 4/6] feat(graphql): makes submit_workflow async --- .vscode/settings.json | 5 ++++- Dockerfile | 2 +- pyproject.toml | 2 ++ src/copier_template/pyproject.toml.jinja | 2 ++ src/copier_template/src/README.md.jinja | 2 +- .../submit_workflow.py.jinja | 4 ++-- .../notebooks/notebook_example.ipynb.jinja | 2 +- .../tests/test_submit_to_graphql.py.jinja | 17 ++++++++++------- .../submit_workflow.py | 4 ++-- .../notebooks/notebook_division.ipynb | 2 +- .../notebooks/notebook_example.ipynb | 6 +++--- tests/test_submit_to_graphql.py | 19 ++++++++++++------- uv.lock | 17 +++++++++++++++++ 13 files changed, 58 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7ac45f8..fb1f0e8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,8 @@ "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", }, - "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env" + "python.envFile": "${workspaceFolder}/workspaces/python-interface-to-workflows/src/.env", + "python.testing.pytestArgs": [ + "tests" + ] } diff --git a/Dockerfile b/Dockerfile index 2a69ffb..0736493 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python # Sync the project without its dev dependencies RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-dev --no-editable --managed-python + uv sync --locked --no-editable --no-dev --managed-python # The runtime stage copies the built venv into a runtime container FROM ubuntu:resolute AS runtime diff --git a/pyproject.toml b/pyproject.toml index 3552c32..50f1105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +47,7 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] [project.scripts] diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index 9eec731..3d5c5bb 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -24,6 +24,7 @@ dependencies = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" @@ -46,6 +47,7 @@ dev = [ "h5py", "dotenv", "python-keycloak", + "pytest-asyncio", ] diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 828529c..23da404 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -2,7 +2,7 @@ 1. run "uv lock" to generate the uv.lock file 2. Create .env in this folder (with the path src/.env) containing the following variables: -HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster) +HOST=https://workflows.diamond.ac.uk/graphql (to submit to the production cluster) DEFAULT_IMAGE= (usually python 3.10) VISIT= (the Visit you wish to run the template on) TOKEN= diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index a2dfed9..e2d353d 100644 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -9,7 +9,7 @@ from hera.workflows import Workflow from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable -def submit_workflow(w: Workflow): +async def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable() @@ -34,7 +34,7 @@ mutation Submit($visit: VisitInput!, $manifest: String!) { } } """) - result = client.execute( + result = await client.execute_async( mutation, variable_values={ "visit": { diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja index 41e2409..f5c3c68 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -222,7 +222,7 @@ "source": [ "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 265d189..2427bd4 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,27 +1,30 @@ -from unittest.mock import MagicMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch from {{project_name}}.submit_workflow import submit_workflow +@pytest.mark.asyncio @patch("{{project_name}}.submit_workflow.os.environ.get") @patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") @patch("{{project_name}}.submit_workflow.Workflow") @patch("{{project_name}}.submit_workflow.set_token_env_variable") @patch("{{project_name}}.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, +async def test_submit_workflow_to_graphql( + mock_client: AsyncMock, mock_key: MagicMock, mock_workflow: MagicMock, mock_load_env: MagicMock, mock_os_get: MagicMock, ): - mock_instance = MagicMock() + mock_instance = AsyncMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) + mock_instance.execute_async = AsyncMock( + return_value={"submitWorkflow": {"name": "workflow123"}} + ) + await submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() + mock_instance.execute_async.assert_called_once() mock_workflow.to_yaml.assert_called_once() mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py index da3422a..6a668c8 100644 --- a/src/python_interface_to_workflows/submit_workflow.py +++ b/src/python_interface_to_workflows/submit_workflow.py @@ -8,7 +8,7 @@ from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable -def submit_workflow(w: Workflow): +async def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) @@ -33,7 +33,7 @@ def submit_workflow(w: Workflow): } } """) - result = client.execute( + result = await client.execute_async( mutation, variable_values={ "visit": { diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 107824e..4d0e147 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -88,7 +88,7 @@ "source": [ "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb index 4d3b47f..1d7bfe4 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -223,13 +223,13 @@ "source": [ "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow(w)" + "await submit_workflow(w)" ] } ], "metadata": { "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", + "display_name": "python-interface-to-workflows (broken)", "language": "python", "name": "python3" }, @@ -243,7 +243,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.15" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index 1eb8020..3a84665 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -1,27 +1,32 @@ -from unittest.mock import MagicMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest from python_interface_to_workflows.submit_workflow import submit_workflow +@pytest.mark.asyncio @patch("python_interface_to_workflows.submit_workflow.os.environ.get") @patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") @patch("python_interface_to_workflows.submit_workflow.Workflow") @patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") @patch("python_interface_to_workflows.submit_workflow.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, +async def test_submit_workflow_to_graphql( + mock_client: AsyncMock, mock_key: MagicMock, mock_workflow: MagicMock, mock_load_env: MagicMock, mock_os_get: MagicMock, ): - mock_instance = MagicMock() + mock_instance = AsyncMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow(mock_workflow) + mock_instance.execute_async = AsyncMock( + return_value={"submitWorkflow": {"name": "workflow123"}} + ) + await submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() + mock_instance.execute_async.assert_called_once() mock_workflow.to_yaml.assert_called_once() mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/uv.lock b/uv.lock index 40041c5..149452e 100644 --- a/uv.lock +++ b/uv.lock @@ -1715,6 +1715,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0" @@ -1763,6 +1776,7 @@ dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, { name = "pyyaml" }, { name = "requests" }, @@ -1779,6 +1793,7 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, { name = "pyyaml" }, @@ -1796,6 +1811,7 @@ requires-dist = [ { name = "hera" }, { name = "numpy" }, { name = "pillow" }, + { name = "pytest-asyncio" }, { name = "python-keycloak" }, { name = "pyyaml" }, { name = "requests" }, @@ -1811,6 +1827,7 @@ dev = [ { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "python-keycloak" }, { name = "pyyaml" }, From 35537bfaf0c91343b41767d22d76f83c917eb32d Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 30 Jul 2026 09:56:17 +0000 Subject: [PATCH 5/6] feat(CI): adds auto-updating image --- .github/workflows/_update_image.yml | 51 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 3 ++ 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/_update_image.yml diff --git a/.github/workflows/_update_image.yml b/.github/workflows/_update_image.yml new file mode 100644 index 0000000..022f244 --- /dev/null +++ b/.github/workflows/_update_image.yml @@ -0,0 +1,51 @@ +name: Update Docker Image + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Generate Image Name + run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-image >> $GITHUB_ENV + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Metadata + id: meta + uses: docker/metadata-action@v6.1.0 + with: + images: ${{ env.IMAGE_REPOSITORY }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=python-interface-to-workflows@v?(.+),group=1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.0.0 + with: + driver-opts: network=host + + - name: Build Image + uses: docker/build-push-action@v6.18.0 + with: + context: . + push: ${{ github.event_name == 'push' }} + load: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f576ab..13ffd5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: commit-lint: uses: ./.github/workflows/_commit_msg.yml + updateimage: + uses: ./.github/workflows/_update_image.yml + test: strategy: matrix: From c23fa23edf0c4371f138aea9643bc58f17de0d43 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 30 Jul 2026 10:59:33 +0000 Subject: [PATCH 6/6] docs(copier): updates copier template and improves example --- scripts/makecopiercorrect.sh | 7 +- .../templates/example.txt.jinja | 41 ++-- .../templates/example_in_image.txt.jinja | 198 ------------------ .../create_example_template.py.jinja | 60 ++++-- ...ple_template_within_default_image.py.jinja | 174 --------------- ...nja => notebook_image_example.ipynb.jinja} | 109 ++++++---- .../auth/keycloak_checker.py | 2 +- .../templates/example.txt | 41 ++-- .../templates/example_in_image.txt | 196 ----------------- .../create_example_template.py | 60 ++++-- ...e_example_template_within_default_image.py | 172 --------------- .../notebooks/notebook_division.ipynb | 4 +- ...ple.ipynb => notebook_image_example.ipynb} | 97 ++++++--- 13 files changed, 256 insertions(+), 905 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja delete mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja rename src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/{notebook_example.ipynb.jinja => notebook_image_example.ipynb.jinja} (78%) delete mode 100644 src/python_interface_to_workflows/templates/example_in_image.txt delete mode 100644 src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py rename src/python_interface_to_workflows/workflow_definitions/notebooks/{notebook_example.ipynb => notebook_image_example.ipynb} (80%) diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 55b5d63..fd9765e 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -11,6 +11,7 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" @@ -18,6 +19,7 @@ do sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" @@ -30,12 +32,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" @@ -48,13 +52,14 @@ do sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" - + sed -i 's/python_interface_to_workflows/{{project_name}}/g' "$file" sed -i '1i{% raw %}' "$file" echo '{% endraw %}' >> "$file" sed -i \ -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + -e 's/{{project_name}}/{% endraw %}{{project_name}}{% raw %}/g' \ "$file" mv "$file" "$file.jinja" diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja index 78cb885..b017cf1 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja @@ -13,12 +13,12 @@ metadata: workflows.diamond.ac.uk/science-group-examples: 'true' spec: entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' templates: - name: workflowentry dag: tasks: - - name: install - template: install-dependencies - name: params template: generate-parameters arguments: @@ -34,7 +34,7 @@ spec: - name: tiff value: 'True' - name: create-image - depends: install && params + depends: params template: create-image withParam: '{{tasks.params.outputs.parameters.out-parameters}}' arguments: @@ -54,22 +54,6 @@ spec: parameters: - name: paths value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - name: generate-parameters inputs: parameters: @@ -84,7 +68,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -129,7 +113,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -163,7 +147,7 @@ spec: with open(f'/tmp/{extension}-path.json', 'w') as f: json.dump(path, f) command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp @@ -178,7 +162,7 @@ spec: archive: none: {} script: - image: python:3.10 + image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image source: |- import os import sys @@ -200,10 +184,19 @@ spec: f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) print('done') command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows volumeClaimTemplates: - metadata: name: tmpdir diff --git a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja deleted file mode 100644 index 2606a5f..0000000 --- a/src/copier_template/src/{{ project_name }}/templates/example_in_image.txt.jinja +++ /dev/null @@ -1,198 +0,0 @@ -{% raw %} -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example-in-image - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index f438048..d3b03e3 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -1,8 +1,13 @@ {% raw %} +import json +import os + +from hera.shared import global_config from hera.workflows import ( DAG, Artifact, Parameter, + Script, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -10,20 +15,9 @@ from hera.workflows import ( from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) ) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) @script( @@ -67,7 +61,7 @@ def generate_parameters( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -117,7 +111,7 @@ def create_image( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", @@ -145,10 +139,37 @@ def to_hdf5(paths: str): with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": { + "cpu": "1", + "memory": "1Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", @@ -159,7 +180,6 @@ example.yaml""", volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: with DAG(name="workflowentry"): - install = install_dependencies(name="install") params = generate_parameters( name="params", arguments={ @@ -176,9 +196,9 @@ example.yaml""", "paths": makeimages.get_parameter("out-paths"), } ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore + params >> makeimages >> makehdf5 # pyright: ignore -with open("example.txt", "w") as div: +with open("src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] {% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja deleted file mode 100644 index 3e604a5..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template_within_default_image.py.jinja +++ /dev/null @@ -1,174 +0,0 @@ -{% raw %} -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example-in-image", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore -with open("example_in_image.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] -{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja similarity index 78% rename from src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja rename to src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja index f5c3c68..8eebe05 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_image_example.ipynb.jinja @@ -1,4 +1,5 @@ -{% raw %}{ +{% raw %} +{ "cells": [ { "cell_type": "markdown", @@ -16,14 +17,18 @@ { "cell_type": "code", "execution_count": null, - "id": "494174ef", + "id": "c1d5a92b", "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", + "from hera.shared import global_config\n", "from hera.workflows import (\n", " DAG,\n", " Artifact,\n", " Parameter,\n", + " Script,\n", " Volume,\n", " Workflow,\n", " script, # pyright: ignore[reportUnknownVariableType]\n", @@ -31,22 +36,18 @@ "from hera.workflows import models as m\n", "from hera.workflows.archive import NoneArchiveStrategy\n", "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=\"ghcr.io/diamondlightsource/python-interface-to-workflows-image\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", " command=[\"python\"],\n", " outputs=Parameter(\n", @@ -84,11 +85,18 @@ " image_params for image_params in params if image_params is not None\n", " ]\n", " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", + " json.dump(params_to_write, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ec312fb", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=[\n", " Parameter(\n", @@ -134,11 +142,18 @@ " path = f\"/tmp/{extension}-image.{extension}\"\n", " image.save(path)\n", " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", + " json.dump(path, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff25236c", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=Artifact(\n", " name=\"hdf5output\",\n", @@ -162,11 +177,29 @@ " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", " f\"image_{i}\", data=arr, dtype=arr.dtype\n", " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", - " name=\"hera-example-\",\n", + " print(\"done\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c93a15d1", + "metadata": {}, + "outputs": [], + "source": [ + "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " },\n", + " \"requests\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " }}}]}),\n", + " tolerations=[\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", + " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", " kind=\"WorkflowTemplate\",\n", @@ -180,7 +213,6 @@ " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", ") as w:\n", " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", " params = generate_parameters(\n", " name=\"params\",\n", " arguments={\n", @@ -197,9 +229,7 @@ " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", " }\n", " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" + " params >> makeimages >> makehdf5 # pyright: ignore" ] }, { @@ -210,7 +240,7 @@ "outputs": [], "source": [ "\n", - "with open(\"example.txt\", \"w\") as div:\n", + "with open(\"src/{% endraw %}{{project_name}}{% raw %}/templates/example.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, @@ -219,7 +249,8 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, - "source": [ + "outputs": [], + "source": [ "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", "await submit_workflow(w)" @@ -228,7 +259,7 @@ ], "metadata": { "kernelspec": { - "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", + "display_name": "{% endraw %}{{repo_name}}{% raw %} (broken)", "language": "python", "name": "python3" }, @@ -242,7 +273,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.15" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py index 6f6ff11..e024fa0 100644 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ b/src/python_interface_to_workflows/auth/keycloak_checker.py @@ -70,7 +70,7 @@ def set_token_env_variable(staging: bool) -> str: ), ) try: - expire_time = int(token_info["exp"]) + 1800 + expire_time = int(token_info["exp"]) + 1500 dotenv.set_key("src/.env", "EXPIRY", str(expire_time)) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt index 1ad8e0e..9dedc96 100644 --- a/src/python_interface_to_workflows/templates/example.txt +++ b/src/python_interface_to_workflows/templates/example.txt @@ -12,12 +12,12 @@ metadata: workflows.diamond.ac.uk/science-group-examples: 'true' spec: entrypoint: workflowentry + podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu": + "1", "memory": "1Gi"}, "requests": {"cpu": "1", "memory": "1Gi"}}}]}' templates: - name: workflowentry dag: tasks: - - name: install - template: install-dependencies - name: params template: generate-parameters arguments: @@ -33,7 +33,7 @@ spec: - name: tiff value: 'True' - name: create-image - depends: install && params + depends: params template: create-image withParam: '{{tasks.params.outputs.parameters.out-parameters}}' arguments: @@ -53,22 +53,6 @@ spec: parameters: - name: paths value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - name: generate-parameters inputs: parameters: @@ -83,7 +67,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -128,7 +112,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -162,7 +146,7 @@ spec: with open(f'/tmp/{extension}-path.json', 'w') as f: json.dump(path, f) command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp @@ -177,7 +161,7 @@ spec: archive: none: {} script: - image: python:3.10 + image: ghcr.io/matt-carre/python-interface-to-workflows-default-image source: |- import os import sys @@ -199,10 +183,19 @@ spec: f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) print('done') command: - - /tmp/venv/bin/python + - python volumeMounts: - name: tmpdir mountPath: /tmp + tolerations: + - effect: NoSchedule + key: nodetype + operator: Equal + value: gpu + - effect: NoSchedule + key: nodegroup + operator: Equal + value: workflows volumeClaimTemplates: - metadata: name: tmpdir diff --git a/src/python_interface_to_workflows/templates/example_in_image.txt b/src/python_interface_to_workflows/templates/example_in_image.txt deleted file mode 100644 index ac6be98..0000000 --- a/src/python_interface_to_workflows/templates/example_in_image.txt +++ /dev/null @@ -1,196 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: hera-example-in-image - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: ghcr.io/matt-carre/python-interface-to-workflows-default-image - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 013387a..804351a 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -1,7 +1,12 @@ +import json +import os + +from hera.shared import global_config from hera.workflows import ( DAG, Artifact, Parameter, + Script, Volume, Workflow, script, # pyright: ignore[reportUnknownVariableType] @@ -9,20 +14,9 @@ from hera.workflows import models as m from hera.workflows.archive import NoneArchiveStrategy - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], +global_config.set_class_defaults( # pyright: ignore + Script, image=str(os.environ.get("DEFAULT_IMAGE")) ) -def install_dependencies(): - import subprocess - - print("creating venv") - - subprocess.check_call(["python", "-m", "venv", "/tmp/venv"]) - subprocess.check_call( - ["/tmp/venv/bin/pip", "install", "pillow", "h5py", "numpy", "hera"] - ) @script( @@ -66,7 +60,7 @@ def generate_parameters( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -116,7 +110,7 @@ def create_pattern( @script( - command=["/tmp/venv/bin/python"], + command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", @@ -144,10 +138,37 @@ def to_hdf5(paths: str): with Workflow( - name="hera-example", # when running on argo this should be generate_name: ...- + pod_spec_patch=json.dumps( + { + "containers": [ + { + "name": "main", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": { + "cpu": "1", + "memory": "1Gi", + }, + }, + } + ] + } + ), + tolerations=[ + m.Toleration( + key="nodetype", operator="Equal", value="gpu", effect="NoSchedule" + ), + m.Toleration( + key="nodegroup", operator="Equal", value="workflows", effect="NoSchedule" + ), + ], + name="hera-example", entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql + kind="WorkflowTemplate", labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", @@ -158,7 +179,6 @@ def to_hdf5(paths: str): volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: with DAG(name="workflowentry"): - install = install_dependencies(name="install") params = generate_parameters( name="params", arguments={ @@ -175,8 +195,8 @@ def to_hdf5(paths: str): "paths": makeimages.get_parameter("out-paths"), } ) - [install, params] >> makeimages >> makehdf5 # pyright: ignore + params >> makeimages >> makehdf5 # pyright: ignore -with open("example.txt", "w") as div: +with open("src/python_interface_to_workflows/templates/example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py deleted file mode 100644 index 779921c..0000000 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template_within_default_image.py +++ /dev/null @@ -1,172 +0,0 @@ -import os - -from hera.shared import global_config -from hera.workflows import ( - DAG, - Artifact, - Parameter, - Script, - Volume, - Workflow, - script, # pyright: ignore[reportUnknownVariableType] -) -from hera.workflows import models as m -from hera.workflows.archive import NoneArchiveStrategy - -global_config.set_class_defaults( # pyright: ignore - Script, image=str(os.environ.get("DEFAULT_IMAGE")) -) - - -@script( - command=["python"], - outputs=Parameter( - name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") - ), - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], -) -def generate_parameters( - png: str, - jpg: str, - jpeg: str, - tif: str, - tiff: str, -): - import json - - params: list[dict[str, int | list[int] | str] | None] = [ - {"width": 500, "height": 500, "weights": [255, 1, 100], "extension": "png"} - if png.lower() == "true" - else None, - {"width": 600, "height": 200, "weights": [100, 150, 100], "extension": "jpg"} - if jpg.lower() == "true" - else None, - {"width": 300, "height": 400, "weights": [100, 150, 100], "extension": "jpeg"} - if jpeg.lower() == "true" - else None, - {"width": 300, "height": 200, "weights": [230, 100, 1], "extension": "tif"} - if tif.lower() == "true" - else None, - {"width": 200, "height": 300, "weights": [230, 100, 1], "extension": "tiff"} - if tiff.lower() == "true" - else None, - ] - params_to_write: list[dict[str, int | list[int] | str]] = [ - image_params for image_params in params if image_params is not None - ] - with open("/tmp/parameters.json", "w") as f: - json.dump(params_to_write, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=[ - Parameter( - name="out-paths", - value_from=m.ValueFrom( - path="/tmp/{{inputs.parameters.extension}}-path.json" - ), - ), - Artifact( - name="{{inputs.parameters.extension}}-image", - path="/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}", - archive=NoneArchiveStrategy(), - ), - ], -) -def create_image( - width: int, height: int, weights: tuple[int, int, int], extension: str -): - import json - - from PIL import Image - - def create_pattern( - width: int, - height: int, - weights: tuple[int, int, int], - ) -> Image.Image: - print(f"width: {width}") - print(f"height: {height}") - print(f"RBG weights: {weights}") - image = Image.new("RGB", (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript] - (i + j * 50) % weights[0], - weights[1], - (i * 300 + j) % weights[2], - ) - return image - - image = create_pattern(width, height, weights) - path = f"/tmp/{extension}-image.{extension}" - image.save(path) - with open(f"/tmp/{extension}-path.json", "w") as f: - json.dump(path, f) - - -@script( - command=["python"], - volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], - outputs=Artifact( - name="hdf5output", - path="/tmp/images.hdf5", - archive=NoneArchiveStrategy(), - ), -) -def to_hdf5(paths: str): - - import h5py # pyright: ignore[reportMissingTypeStubs] - import numpy as np - from PIL import Image - - print("creating hdf5 file") - with h5py.File("/tmp/images.hdf5", "w") as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f"Got {path}") - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset( # pyright: ignore[reportUnknownMemberType] - f"image_{i}", data=arr, dtype=arr.dtype - ) - print("done") - - -with Workflow( - name="hera-example-in-image", - entrypoint="workflowentry", - api_version="argoproj.io/v1alpha1", - kind="WorkflowTemplate", - labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, - annotations={ - "workflows.argoproj.io/title": "example remade via hera", - "workflows.argoproj.io/description": """Replicates the functionality of -example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", - }, - volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), -) as w: - with DAG(name="workflowentry"): - params = generate_parameters( - name="params", - arguments={ - "png": "True", - "jpg": "True", - "jpeg": "True", - "tif": "True", - "tiff": "True", - }, - ) - makeimages = create_image(with_param=params.get_parameter("out-parameters")) - makehdf5 = to_hdf5( - arguments={ - "paths": makeimages.get_parameter("out-paths"), - } - ) - params >> makeimages >> makehdf5 # pyright: ignore -with open("example_in_image.txt", "w") as div: - div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index 4d0e147..de4001c 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -64,8 +64,8 @@ " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", ") as w:\n", " with Steps(name=\"divide\"):\n", - " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", - "\n" + " do_division(name=\"first\", arguments={\"a\":\"{{.Files.get '/notebooks/a.json'}}\",\n", + " \"b\":\"{{.Files.get '/notebooks/b.json'}}\"}) # file.get here\n" ] }, { diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb similarity index 80% rename from src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb rename to src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb index 1d7bfe4..ef7becc 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_image_example.ipynb @@ -16,14 +16,18 @@ { "cell_type": "code", "execution_count": null, - "id": "494174ef", + "id": "c1d5a92b", "metadata": {}, "outputs": [], "source": [ + "import json\n", + "\n", + "from hera.shared import global_config\n", "from hera.workflows import (\n", " DAG,\n", " Artifact,\n", " Parameter,\n", + " Script,\n", " Volume,\n", " Workflow,\n", " script, # pyright: ignore[reportUnknownVariableType]\n", @@ -31,22 +35,18 @@ "from hera.workflows import models as m\n", "from hera.workflows.archive import NoneArchiveStrategy\n", "\n", - "\n", - "@script(\n", - " command=[\"python\"],\n", - " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", - ")\n", - "def install_dependencies():\n", - " import subprocess\n", - "\n", - " print(\"creating venv\")\n", - "\n", - " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", - " subprocess.check_call(\n", - " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", - " )\n", - "\n", - "\n", + "global_config.set_class_defaults( # pyright: ignore\n", + " Script, image=str(os.environ.get(\"DEFAULT_IMAGE\"))\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", " command=[\"python\"],\n", " outputs=Parameter(\n", @@ -84,11 +84,18 @@ " image_params for image_params in params if image_params is not None\n", " ]\n", " with open(\"/tmp/parameters.json\", \"w\") as f:\n", - " json.dump(params_to_write, f)\n", - "\n", - "\n", + " json.dump(params_to_write, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ec312fb", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=[\n", " Parameter(\n", @@ -134,11 +141,18 @@ " path = f\"/tmp/{extension}-image.{extension}\"\n", " image.save(path)\n", " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", - " json.dump(path, f)\n", - "\n", - "\n", + " json.dump(path, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff25236c", + "metadata": {}, + "outputs": [], + "source": [ "@script(\n", - " command=[\"/tmp/venv/bin/python\"],\n", + " command=[\"python\"],\n", " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", " outputs=Artifact(\n", " name=\"hdf5output\",\n", @@ -162,10 +176,28 @@ " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", " f\"image_{i}\", data=arr, dtype=arr.dtype\n", " )\n", - " print(\"done\")\n", - "\n", - "\n", - "with Workflow(\n", + " print(\"done\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c93a15d1", + "metadata": {}, + "outputs": [], + "source": [ + "with Workflow(pod_spec_patch=json.dumps({\"containers\":\n", + " [{\"name\":\"main\",\n", + " \"resources\":\n", + " {\"limits\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " },\n", + " \"requests\":{\"cpu\":\"1\",\n", + " \"memory\":\"1Gi\",\n", + " }}}]}),\n", + " tolerations=[\n", + " m.Toleration(key=\"nodetype\",operator=\"Equal\",value=\"gpu\",effect=\"NoSchedule\"),\n", + " m.Toleration(key=\"nodegroup\",operator=\"Equal\",value=\"workflows\",effect=\"NoSchedule\")],\n", " name=\"hera-example\",\n", " entrypoint=\"workflowentry\",\n", " api_version=\"argoproj.io/v1alpha1\",\n", @@ -180,7 +212,6 @@ " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", ") as w:\n", " with DAG(name=\"workflowentry\"):\n", - " install = install_dependencies(name=\"install\")\n", " params = generate_parameters(\n", " name=\"params\",\n", " arguments={\n", @@ -197,9 +228,7 @@ " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", " }\n", " )\n", - " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", - "\n", - "\n" + " params >> makeimages >> makehdf5 # pyright: ignore" ] }, { @@ -210,7 +239,7 @@ "outputs": [], "source": [ "\n", - "with open(\"../../templates/example_from_jupyter.txt\", \"w\") as div:\n", + "with open(\"src/python_interface_to_workflows/templates/example.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] },