diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 71710a6..cfa6f9e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -39,7 +39,16 @@ "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-python.debugpy", + "ms-python.vscode-python-envs", + "ms-python.vscode-pylance", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.jupyter", + "ms-azuretools.vscode-containers", + "ms-toolsai.jupyter-keymap" ] } }, diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 973bd00..55b5d63 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -6,6 +6,7 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/* do + [[ -d "$file" ]] && continue [[ $file == *.jinja ]] && continue sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" @@ -21,9 +22,28 @@ do mv "$file" "$file.jinja" done +for file in "../../copier_template/src/{{ project_name }}/workflow_definitions/notebooks"/* +do + [[ -d "$file" ]] && continue + [[ $file == *.jinja ]] && continue + + sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" + sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + + sed -i '1i{% raw %}' "$file" + echo '{% endraw %}' >> "$file" + sed -i \ + -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ + -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + "$file" + + + mv "$file" "$file.jinja" +done for file in "../../copier_template/src/{{ project_name }}/templates"/* do + [[ -d "$file" ]] && continue [[ $file == *.jinja ]] && continue sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index 960a7d8..662bf8e 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -2,8 +2,8 @@ cd src/python_interface_to_workflows/workflow_definitions for file in * do +[[ -d "$file" ]] && continue uv run "$file" done -mv *.yaml ../templates/ mv *.txt ../templates/ git add -u ../templates/ diff --git a/src/copier_template/.devcontainer/devcontainer.json b/src/copier_template/.devcontainer/devcontainer.json index 71710a6..cfa6f9e 100644 --- a/src/copier_template/.devcontainer/devcontainer.json +++ b/src/copier_template/.devcontainer/devcontainer.json @@ -39,7 +39,16 @@ "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-python.debugpy", + "ms-python.vscode-python-envs", + "ms-python.vscode-pylance", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.jupyter", + "ms-azuretools.vscode-containers", + "ms-toolsai.jupyter-keymap" ] } }, diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index f324322..c4d14a7 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -2,8 +2,8 @@ cd src/{{project_name}}/workflow_definitions for file in * do +[[ -d "$file" ]] && continue uv run "$file" done -mv *.yaml ../templates/ -mv *.txt ../templates/ -git add -u ../templates/ +mv *.txt src/{{project_name}}/templates/ +git add -u src/{{project_name}}/templates/ diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index 597599a..a2dfed9 100644 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -12,7 +12,7 @@ from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_toke def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable(True) + token: str = set_token_env_variable() host: str = os.environ.get("HOST") # pyright:ignore visit: str = os.environ.get("VISIT") # pyright:ignore diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja new file mode 100644 index 0000000..0f829f6 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -0,0 +1,251 @@ +{% raw %}{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_example.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows import (\n", + " DAG,\n", + " Artifact,\n", + " Parameter,\n", + " Volume,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def install_dependencies():\n", + " import subprocess\n", + "\n", + " print(\"creating venv\")\n", + "\n", + " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", + " subprocess.check_call(\n", + " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", + " )\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " outputs=Parameter(\n", + " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", + " ),\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def generate_parameters(\n", + " png: str,\n", + " jpg: str,\n", + " jpeg: str,\n", + " tif: str,\n", + " tiff: str,\n", + "):\n", + " import json\n", + "\n", + " params: list[dict[str, int | list[int] | str] | None] = [\n", + " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", + " if png.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", + " if jpg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", + " if jpeg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", + " if tif.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", + " if tiff.lower() == \"true\"\n", + " else None,\n", + " ]\n", + " params_to_write: list[dict[str, int | list[int] | str]] = [\n", + " image_params for image_params in params if image_params is not None\n", + " ]\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(params_to_write, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-paths\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", + " ),\n", + " ),\n", + " Artifact(\n", + " name=\"{{inputs.parameters.extension}}-image\",\n", + " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ],\n", + ")\n", + "def create_image(\n", + " width: int, height: int, weights: tuple[int, int, int], extension: str\n", + "):\n", + " import json\n", + "\n", + " from PIL import Image\n", + "\n", + " def create_pattern(\n", + " width: int,\n", + " height: int,\n", + " weights: tuple[int, int, int],\n", + " ) -> Image.Image:\n", + " print(f\"width: {width}\")\n", + " print(f\"height: {height}\")\n", + " print(f\"RBG weights: {weights}\")\n", + " image = Image.new(\"RGB\", (width, height))\n", + " pixels = image.load()\n", + " for i in range(width):\n", + " for j in range(height):\n", + " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", + " (i + j * 50) % weights[0],\n", + " weights[1],\n", + " (i * 300 + j) % weights[2],\n", + " )\n", + " return image\n", + "\n", + " image = create_pattern(width, height, weights)\n", + " path = f\"/tmp/{extension}-image.{extension}\"\n", + " image.save(path)\n", + " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", + " json.dump(path, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=Artifact(\n", + " name=\"hdf5output\",\n", + " path=\"/tmp/images.hdf5\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + ")\n", + "def to_hdf5(paths: str):\n", + "\n", + " import h5py # pyright: ignore[reportMissingTypeStubs]\n", + " import numpy as np\n", + " from PIL import Image\n", + "\n", + " print(\"creating hdf5 file\")\n", + " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", + " for i, path in enumerate(paths):\n", + " path = path.strip('\"')\n", + " print(f\"Got {path}\")\n", + " with Image.open(path) as image:\n", + " arr = np.array(image)\n", + " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", + " f\"image_{i}\", data=arr, dtype=arr.dtype\n", + " )\n", + " print(\"done\")\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", + " },\n", + " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " install = install_dependencies(name=\"install\")\n", + " params = generate_parameters(\n", + " name=\"params\",\n", + " arguments={\n", + " \"png\": \"True\",\n", + " \"jpg\": \"True\",\n", + " \"jpeg\": \"True\",\n", + " \"tif\": \"True\",\n", + " \"tiff\": \"True\",\n", + " },\n", + " )\n", + " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", + " makehdf5 = to_hdf5(\n", + " arguments={\n", + " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", + " }\n", + " )\n", + " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(\"example.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "source": [ + "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", + "\n", + "submit_workflow(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} +{% endraw %} diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index 4b27f07..b960007 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -45,7 +45,7 @@ def test_set_token_env_variable( keycloak.token.return_value = token keycloak.refresh_token.return_value = token keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable(staging) == "fake_token" + assert set_token_env_variable() == "fake_token" mock_token_expired.assert_called_once_with("https://mock.site", port) if return_present: diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 25add52..265d189 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from {{project_name}}.submit_workflow import submit_workflow diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.txt similarity index 100% rename from src/python_interface_to_workflows/templates/divisionyaml.yaml rename to src/python_interface_to_workflows/templates/divisionyaml.txt diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py index 8a57ae1..327618f 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py @@ -45,5 +45,5 @@ def do_division(a: int, b: int): do_division(name="first", arguments={"a": 2, "b": 5}) -with open("divisionyaml.yaml", "w") as div: +with open("divisionyaml.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb new file mode 100644 index 0000000..9113672 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -0,0 +1,116 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_division.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from hera.workflows import (\n", + " Artifact,\n", + " EmptyDirVolume,\n", + " Steps,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "\n", + "\n", + "@script(\n", + " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", + " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", + ")\n", + "def do_division(a: int, b: int):\n", + " div = a / b\n", + " intdiv = a // b\n", + " remain = a % b\n", + " dictionary_of_results = {\n", + " \"divide\": div,\n", + " \"quotient\": intdiv,\n", + " \"remainder\": remain,\n", + " }\n", + " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", + " json.dump(dictionary_of_results, otpt)\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " entrypoint=\"divide\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", + " the remainder, output float, and output string to a json file\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", + ") as w:\n", + " with Steps(name=\"divide\"):\n", + " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"../../templates/division_from_jupyter.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "outputs": [], + "source": [ + "from python_interface_to_workflows.submit_workflow import submit_workflow\n", + "\n", + "submit_workflow(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.-1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb new file mode 100644 index 0000000..c1018ea --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_example.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows import (\n", + " DAG,\n", + " Artifact,\n", + " Parameter,\n", + " Volume,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def install_dependencies():\n", + " import subprocess\n", + "\n", + " print(\"creating venv\")\n", + "\n", + " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", + " subprocess.check_call(\n", + " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", + " )\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " outputs=Parameter(\n", + " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", + " ),\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def generate_parameters(\n", + " png: str,\n", + " jpg: str,\n", + " jpeg: str,\n", + " tif: str,\n", + " tiff: str,\n", + "):\n", + " import json\n", + "\n", + " params: list[dict[str, int | list[int] | str] | None] = [\n", + " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", + " if png.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", + " if jpg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", + " if jpeg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", + " if tif.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", + " if tiff.lower() == \"true\"\n", + " else None,\n", + " ]\n", + " params_to_write: list[dict[str, int | list[int] | str]] = [\n", + " image_params for image_params in params if image_params is not None\n", + " ]\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(params_to_write, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-paths\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", + " ),\n", + " ),\n", + " Artifact(\n", + " name=\"{{inputs.parameters.extension}}-image\",\n", + " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ],\n", + ")\n", + "def create_image(\n", + " width: int, height: int, weights: tuple[int, int, int], extension: str\n", + "):\n", + " import json\n", + "\n", + " from PIL import Image\n", + "\n", + " def create_pattern(\n", + " width: int,\n", + " height: int,\n", + " weights: tuple[int, int, int],\n", + " ) -> Image.Image:\n", + " print(f\"width: {width}\")\n", + " print(f\"height: {height}\")\n", + " print(f\"RBG weights: {weights}\")\n", + " image = Image.new(\"RGB\", (width, height))\n", + " pixels = image.load()\n", + " for i in range(width):\n", + " for j in range(height):\n", + " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", + " (i + j * 50) % weights[0],\n", + " weights[1],\n", + " (i * 300 + j) % weights[2],\n", + " )\n", + " return image\n", + "\n", + " image = create_pattern(width, height, weights)\n", + " path = f\"/tmp/{extension}-image.{extension}\"\n", + " image.save(path)\n", + " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", + " json.dump(path, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=Artifact(\n", + " name=\"hdf5output\",\n", + " path=\"/tmp/images.hdf5\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + ")\n", + "def to_hdf5(paths: str):\n", + "\n", + " import h5py # pyright: ignore[reportMissingTypeStubs]\n", + " import numpy as np\n", + " from PIL import Image\n", + "\n", + " print(\"creating hdf5 file\")\n", + " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", + " for i, path in enumerate(paths):\n", + " path = path.strip('\"')\n", + " print(f\"Got {path}\")\n", + " with Image.open(path) as image:\n", + " arr = np.array(image)\n", + " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", + " f\"image_{i}\", data=arr, dtype=arr.dtype\n", + " )\n", + " print(\"done\")\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " install = install_dependencies(name=\"install\")\n", + " params = generate_parameters(\n", + " name=\"params\",\n", + " arguments={\n", + " \"png\": \"True\",\n", + " \"jpg\": \"True\",\n", + " \"jpeg\": \"True\",\n", + " \"tif\": \"True\",\n", + " \"tiff\": \"True\",\n", + " },\n", + " )\n", + " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", + " makehdf5 = to_hdf5(\n", + " arguments={\n", + " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", + " }\n", + " )\n", + " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(\"../../templates/example_from_jupyter.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "outputs": [], + "source": [ + "from python_interface_to_workflows.submit_workflow import submit_workflow\n", + "\n", + "submit_workflow(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}