From c97c330fa347b2cf26f608994df4717d64b588aa Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 12:40:22 +0000 Subject: [PATCH 1/5] feat(ipynb): adds example copy of notebook --- .../notebook_division.ipynb | 138 +++++++++ .../notebook_example.ipynb | 273 ++++++++++++++++++ 2 files changed, 411 insertions(+) create mode 100644 src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb create mode 100644 src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb new file mode 100644 index 0000000..6808de9 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_division.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from hera.workflows import (\n", + " Artifact,\n", + " EmptyDirVolume,\n", + " Steps,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "\n", + "\n", + "@script(\n", + " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", + " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", + ")\n", + "def do_division(a: int, b: int):\n", + " div = a / b\n", + " intdiv = a // b\n", + " remain = a % b\n", + " dictionary_of_results = {\n", + " \"divide\": div,\n", + " \"quotient\": intdiv,\n", + " \"remainder\": remain,\n", + " }\n", + " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", + " json.dump(dictionary_of_results, otpt)\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " entrypoint=\"divide\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", + " the remainder, output float, and output string to a json file\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", + ") as w:\n", + " with Steps(name=\"divide\"):\n", + " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "expiry_str: '1784802224' 1784801151.7235122\n" + ] + }, + { + "ename": "ValueError", + "evalue": "invalid literal for int() with base 10: \"'1784802224'\"", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m python_interface_to_workflows.submit_to_argo \u001b[38;5;28;01mimport\u001b[39;00m submit_workflow_to_argo\n\u001b[32m 2\u001b[39m \n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m submit_workflow_to_argo(w)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/submit_to_argo.py:13\u001b[39m, in \u001b[36msubmit_workflow_to_argo\u001b[39m\u001b[34m(w)\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msubmit_workflow_to_argo\u001b[39m(w: Workflow):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[30;43mset_token_env_variable\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstaging\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43;01mTrue\u001b[39;49;00m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 14\u001b[39m dotenv.load_dotenv(dotenv_path=\u001b[33m\"\u001b[39m\u001b[33msrc/.env\u001b[39m\u001b[33m\"\u001b[39m, override=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 15\u001b[39m w.namespace = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mNAMESPACE\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/keycloak_checker.py:39\u001b[39m, in \u001b[36mset_token_env_variable\u001b[39m\u001b[34m(staging)\u001b[39m\n\u001b[32m 31\u001b[39m code_challenge, code_challenge_method = generate_code_challenge(code_verifier)\n\u001b[32m 32\u001b[39m auth_url = keycloak_openid.auth_url(\n\u001b[32m 33\u001b[39m redirect_uri=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mport\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 34\u001b[39m scope=\u001b[33m\"\u001b[39m\u001b[33mopenid posix-uid profile email fedid\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 37\u001b[39m code_challenge_method=code_challenge_method,\n\u001b[32m 38\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mmatch\u001b[39;00m \u001b[30;43mopen_auth_url\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mauth_url\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mport\u001b[39;49m\u001b[30;43m)\u001b[39;49m:\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mcase\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 41\u001b[39m token: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = ( \u001b[38;5;66;03m# pyright: ignore[reportUnknownVariableType]\u001b[39;00m\n\u001b[32m 42\u001b[39m keycloak_openid.token( \u001b[38;5;66;03m# pyright: ignore[reportUnknownMemberType]\u001b[39;00m\n\u001b[32m 43\u001b[39m grant_type=\u001b[33m\"\u001b[39m\u001b[33mauthorization_code\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 47\u001b[39m )\n\u001b[32m 48\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/open_auth_url.py:36\u001b[39m, in \u001b[36mopen_auth_url\u001b[39m\u001b[34m(auth_url, port)\u001b[39m\n\u001b[32m 34\u001b[39m expiry_str: \u001b[38;5;28mstr\u001b[39m = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mEXPIRY\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;66;03m# pyright: ignore\u001b[39;00m\n\u001b[32m 35\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mexpiry_str: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpiry_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m, \u001b[38;5;28mstr\u001b[39m(time.time()))\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (expiry_str == \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[30;43mint\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mexpiry_str\u001b[39;49m\u001b[30;43m)\u001b[39;49m) <= \u001b[38;5;28mfloat\u001b[39m(time.time()):\n\u001b[32m 37\u001b[39m httpd = _ReusingHTTPServer((\u001b[33m\"\u001b[39m\u001b[33mlocalhost\u001b[39m\u001b[33m\"\u001b[39m, port), CallbackHandler)\n\u001b[32m 38\u001b[39m webbrowser.open(auth_url)\n", + "\u001b[31mValueError\u001b[39m: invalid literal for int() with base 10: \"'1784802224'\"" + ] + } + ], + "source": [ + "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", + "\n", + "submit_workflow_to_argo(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb new file mode 100644 index 0000000..fde4ccd --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_example.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows import (\n", + " DAG,\n", + " Artifact,\n", + " Parameter,\n", + " Volume,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def install_dependencies():\n", + " import subprocess\n", + "\n", + " print(\"creating venv\")\n", + "\n", + " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", + " subprocess.check_call(\n", + " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", + " )\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " outputs=Parameter(\n", + " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", + " ),\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def generate_parameters(\n", + " png: str,\n", + " jpg: str,\n", + " jpeg: str,\n", + " tif: str,\n", + " tiff: str,\n", + "):\n", + " import json\n", + "\n", + " params: list[dict[str, int | list[int] | str] | None] = [\n", + " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", + " if png.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", + " if jpg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", + " if jpeg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", + " if tif.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", + " if tiff.lower() == \"true\"\n", + " else None,\n", + " ]\n", + " params_to_write: list[dict[str, int | list[int] | str]] = [\n", + " image_params for image_params in params if image_params is not None\n", + " ]\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(params_to_write, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-paths\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", + " ),\n", + " ),\n", + " Artifact(\n", + " name=\"{{inputs.parameters.extension}}-image\",\n", + " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ],\n", + ")\n", + "def create_image(\n", + " width: int, height: int, weights: tuple[int, int, int], extension: str\n", + "):\n", + " import json\n", + "\n", + " from PIL import Image\n", + "\n", + " def create_pattern(\n", + " width: int,\n", + " height: int,\n", + " weights: tuple[int, int, int],\n", + " ) -> Image.Image:\n", + " print(f\"width: {width}\")\n", + " print(f\"height: {height}\")\n", + " print(f\"RBG weights: {weights}\")\n", + " image = Image.new(\"RGB\", (width, height))\n", + " pixels = image.load()\n", + " for i in range(width):\n", + " for j in range(height):\n", + " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", + " (i + j * 50) % weights[0],\n", + " weights[1],\n", + " (i * 300 + j) % weights[2],\n", + " )\n", + " return image\n", + "\n", + " image = create_pattern(width, height, weights)\n", + " path = f\"/tmp/{extension}-image.{extension}\"\n", + " image.save(path)\n", + " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", + " json.dump(path, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=Artifact(\n", + " name=\"hdf5output\",\n", + " path=\"/tmp/images.hdf5\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + ")\n", + "def to_hdf5(paths: str):\n", + "\n", + " import h5py # pyright: ignore[reportMissingTypeStubs]\n", + " import numpy as np\n", + " from PIL import Image\n", + "\n", + " print(\"creating hdf5 file\")\n", + " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", + " for i, path in enumerate(paths):\n", + " path = path.strip('\"')\n", + " print(f\"Got {path}\")\n", + " with Image.open(path) as image:\n", + " arr = np.array(image)\n", + " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", + " f\"image_{i}\", data=arr, dtype=arr.dtype\n", + " )\n", + " print(\"done\")\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " install = install_dependencies(name=\"install\")\n", + " params = generate_parameters(\n", + " name=\"params\",\n", + " arguments={\n", + " \"png\": \"True\",\n", + " \"jpg\": \"True\",\n", + " \"jpeg\": \"True\",\n", + " \"tif\": \"True\",\n", + " \"tiff\": \"True\",\n", + " },\n", + " )\n", + " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", + " makehdf5 = to_hdf5(\n", + " arguments={\n", + " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", + " }\n", + " )\n", + " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(\"example.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "expiry_str: '1784802224' 1784801151.7235122\n" + ] + }, + { + "ename": "ValueError", + "evalue": "invalid literal for int() with base 10: \"'1784802224'\"", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m python_interface_to_workflows.submit_to_argo \u001b[38;5;28;01mimport\u001b[39;00m submit_workflow_to_argo\n\u001b[32m 2\u001b[39m \n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m submit_workflow_to_argo(w)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/submit_to_argo.py:13\u001b[39m, in \u001b[36msubmit_workflow_to_argo\u001b[39m\u001b[34m(w)\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msubmit_workflow_to_argo\u001b[39m(w: Workflow):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[30;43mset_token_env_variable\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstaging\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43;01mTrue\u001b[39;49;00m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 14\u001b[39m dotenv.load_dotenv(dotenv_path=\u001b[33m\"\u001b[39m\u001b[33msrc/.env\u001b[39m\u001b[33m\"\u001b[39m, override=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 15\u001b[39m w.namespace = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mNAMESPACE\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/keycloak_checker.py:39\u001b[39m, in \u001b[36mset_token_env_variable\u001b[39m\u001b[34m(staging)\u001b[39m\n\u001b[32m 31\u001b[39m code_challenge, code_challenge_method = generate_code_challenge(code_verifier)\n\u001b[32m 32\u001b[39m auth_url = keycloak_openid.auth_url(\n\u001b[32m 33\u001b[39m redirect_uri=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mport\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 34\u001b[39m scope=\u001b[33m\"\u001b[39m\u001b[33mopenid posix-uid profile email fedid\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 37\u001b[39m code_challenge_method=code_challenge_method,\n\u001b[32m 38\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mmatch\u001b[39;00m \u001b[30;43mopen_auth_url\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mauth_url\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mport\u001b[39;49m\u001b[30;43m)\u001b[39;49m:\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mcase\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 41\u001b[39m token: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = ( \u001b[38;5;66;03m# pyright: ignore[reportUnknownVariableType]\u001b[39;00m\n\u001b[32m 42\u001b[39m keycloak_openid.token( \u001b[38;5;66;03m# pyright: ignore[reportUnknownMemberType]\u001b[39;00m\n\u001b[32m 43\u001b[39m grant_type=\u001b[33m\"\u001b[39m\u001b[33mauthorization_code\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 47\u001b[39m )\n\u001b[32m 48\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/open_auth_url.py:36\u001b[39m, in \u001b[36mopen_auth_url\u001b[39m\u001b[34m(auth_url, port)\u001b[39m\n\u001b[32m 34\u001b[39m expiry_str: \u001b[38;5;28mstr\u001b[39m = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mEXPIRY\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;66;03m# pyright: ignore\u001b[39;00m\n\u001b[32m 35\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mexpiry_str: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpiry_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m, \u001b[38;5;28mstr\u001b[39m(time.time()))\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (expiry_str == \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[30;43mint\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mexpiry_str\u001b[39;49m\u001b[30;43m)\u001b[39;49m) <= \u001b[38;5;28mfloat\u001b[39m(time.time()):\n\u001b[32m 37\u001b[39m httpd = _ReusingHTTPServer((\u001b[33m\"\u001b[39m\u001b[33mlocalhost\u001b[39m\u001b[33m\"\u001b[39m, port), CallbackHandler)\n\u001b[32m 38\u001b[39m webbrowser.open(auth_url)\n", + "\u001b[31mValueError\u001b[39m: invalid literal for int() with base 10: \"'1784802224'\"" + ] + } + ], + "source": [ + "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", + "\n", + "submit_workflow_to_argo(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 467aef28b2febb13a9d6216f75a5a8c5b69ad57e Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 27 Jul 2026 10:07:49 +0000 Subject: [PATCH 2/5] feat(ipynb): adds copier versions of notebooks --- .../notebook_division.ipynb.jinja | 116 ++++++++ .../notebook_example.ipynb.jinja | 251 ++++++++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja create mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja new file mode 100644 index 0000000..c8b4218 --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja @@ -0,0 +1,116 @@ +{% raw %}{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_division.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from hera.workflows import (\n", + " Artifact,\n", + " EmptyDirVolume,\n", + " Steps,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "\n", + "\n", + "@script(\n", + " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", + " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", + ")\n", + "def do_division(a: int, b: int):\n", + " div = a / b\n", + " intdiv = a // b\n", + " remain = a % b\n", + " dictionary_of_results = {\n", + " \"divide\": div,\n", + " \"quotient\": intdiv,\n", + " \"remainder\": remain,\n", + " }\n", + " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", + " json.dump(dictionary_of_results, otpt)\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " entrypoint=\"divide\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", + " the remainder, output float, and output string to a json file\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", + ") as w:\n", + " with Steps(name=\"divide\"):\n", + " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "source": [ + "from {% endraw %}{{project_name}}{% raw %}.submit_to_argo import submit_workflow_to_argo\n", + "\n", + "submit_workflow_to_argo(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja new file mode 100644 index 0000000..aca499c --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja @@ -0,0 +1,251 @@ +{% raw %}{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_example.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "from hera.workflows import (\n", + " DAG,\n", + " Artifact,\n", + " Parameter,\n", + " Volume,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "from hera.workflows.archive import NoneArchiveStrategy\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def install_dependencies():\n", + " import subprocess\n", + "\n", + " print(\"creating venv\")\n", + "\n", + " subprocess.check_call([\"python\", \"-m\", \"venv\", \"/tmp/venv\"])\n", + " subprocess.check_call(\n", + " [\"/tmp/venv/bin/pip\", \"install\", \"pillow\", \"h5py\", \"numpy\", \"hera\"]\n", + " )\n", + "\n", + "\n", + "@script(\n", + " command=[\"python\"],\n", + " outputs=Parameter(\n", + " name=\"out-parameters\", value_from=m.ValueFrom(path=\"/tmp/parameters.json\")\n", + " ),\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + ")\n", + "def generate_parameters(\n", + " png: str,\n", + " jpg: str,\n", + " jpeg: str,\n", + " tif: str,\n", + " tiff: str,\n", + "):\n", + " import json\n", + "\n", + " params: list[dict[str, int | list[int] | str] | None] = [\n", + " {\"width\": 500, \"height\": 500, \"weights\": [255, 1, 100], \"extension\": \"png\"}\n", + " if png.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 600, \"height\": 200, \"weights\": [100, 150, 100], \"extension\": \"jpg\"}\n", + " if jpg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 400, \"weights\": [100, 150, 100], \"extension\": \"jpeg\"}\n", + " if jpeg.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 300, \"height\": 200, \"weights\": [230, 100, 1], \"extension\": \"tif\"}\n", + " if tif.lower() == \"true\"\n", + " else None,\n", + " {\"width\": 200, \"height\": 300, \"weights\": [230, 100, 1], \"extension\": \"tiff\"}\n", + " if tiff.lower() == \"true\"\n", + " else None,\n", + " ]\n", + " params_to_write: list[dict[str, int | list[int] | str]] = [\n", + " image_params for image_params in params if image_params is not None\n", + " ]\n", + " with open(\"/tmp/parameters.json\", \"w\") as f:\n", + " json.dump(params_to_write, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=[\n", + " Parameter(\n", + " name=\"out-paths\",\n", + " value_from=m.ValueFrom(\n", + " path=\"/tmp/{{inputs.parameters.extension}}-path.json\"\n", + " ),\n", + " ),\n", + " Artifact(\n", + " name=\"{{inputs.parameters.extension}}-image\",\n", + " path=\"/tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}}\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + " ],\n", + ")\n", + "def create_image(\n", + " width: int, height: int, weights: tuple[int, int, int], extension: str\n", + "):\n", + " import json\n", + "\n", + " from PIL import Image\n", + "\n", + " def create_pattern(\n", + " width: int,\n", + " height: int,\n", + " weights: tuple[int, int, int],\n", + " ) -> Image.Image:\n", + " print(f\"width: {width}\")\n", + " print(f\"height: {height}\")\n", + " print(f\"RBG weights: {weights}\")\n", + " image = Image.new(\"RGB\", (width, height))\n", + " pixels = image.load()\n", + " for i in range(width):\n", + " for j in range(height):\n", + " pixels[i, j] = ( # pyright: ignore[reportOptionalSubscript]\n", + " (i + j * 50) % weights[0],\n", + " weights[1],\n", + " (i * 300 + j) % weights[2],\n", + " )\n", + " return image\n", + "\n", + " image = create_pattern(width, height, weights)\n", + " path = f\"/tmp/{extension}-image.{extension}\"\n", + " image.save(path)\n", + " with open(f\"/tmp/{extension}-path.json\", \"w\") as f:\n", + " json.dump(path, f)\n", + "\n", + "\n", + "@script(\n", + " command=[\"/tmp/venv/bin/python\"],\n", + " volume_mounts=[m.VolumeMount(name=\"tmpdir\", mount_path=\"/tmp\")],\n", + " outputs=Artifact(\n", + " name=\"hdf5output\",\n", + " path=\"/tmp/images.hdf5\",\n", + " archive=NoneArchiveStrategy(),\n", + " ),\n", + ")\n", + "def to_hdf5(paths: str):\n", + "\n", + " import h5py # pyright: ignore[reportMissingTypeStubs]\n", + " import numpy as np\n", + " from PIL import Image\n", + "\n", + " print(\"creating hdf5 file\")\n", + " with h5py.File(\"/tmp/images.hdf5\", \"w\") as f:\n", + " for i, path in enumerate(paths):\n", + " path = path.strip('\"')\n", + " print(f\"Got {path}\")\n", + " with Image.open(path) as image:\n", + " arr = np.array(image)\n", + " f.create_dataset( # pyright: ignore[reportUnknownMemberType]\n", + " f\"image_{i}\", data=arr, dtype=arr.dtype\n", + " )\n", + " print(\"done\")\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-example-\", # when running on graphql this should be name\n", + " entrypoint=\"workflowentry\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", + "example.yaml\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", + ") as w:\n", + " with DAG(name=\"workflowentry\"):\n", + " install = install_dependencies(name=\"install\")\n", + " params = generate_parameters(\n", + " name=\"params\",\n", + " arguments={\n", + " \"png\": \"True\",\n", + " \"jpg\": \"True\",\n", + " \"jpeg\": \"True\",\n", + " \"tif\": \"True\",\n", + " \"tiff\": \"True\",\n", + " },\n", + " )\n", + " makeimages = create_image(with_param=params.get_parameter(\"out-parameters\"))\n", + " makehdf5 = to_hdf5(\n", + " arguments={\n", + " \"paths\": makeimages.get_parameter(\"out-paths\"),\n", + " }\n", + " )\n", + " [install, params] >> makeimages >> makehdf5 # pyright: ignore\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(\"example.txt\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "source": [ + "from {% endraw %}{{project_name}}{% raw %}.submit_to_argo import submit_workflow_to_argo\n", + "\n", + "submit_workflow_to_argo(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} +{% endraw %} From 91152d2987d692354822be5583bba5d4ee872559 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Tue, 28 Jul 2026 08:50:51 +0000 Subject: [PATCH 3/5] feat(ipynb): clears up notebooks, adds jupyter to dev containers --- .devcontainer/devcontainer.json | 4 ++- .../.devcontainer/devcontainer.json | 4 ++- .../notebook_division.ipynb | 26 ++----------------- .../notebook_example.ipynb | 24 +---------------- 4 files changed, 9 insertions(+), 49 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 71710a6..2fb7099 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -39,7 +39,9 @@ "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter" ] } }, diff --git a/src/copier_template/.devcontainer/devcontainer.json b/src/copier_template/.devcontainer/devcontainer.json index 71710a6..2fb7099 100644 --- a/src/copier_template/.devcontainer/devcontainer.json +++ b/src/copier_template/.devcontainer/devcontainer.json @@ -39,7 +39,9 @@ "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter" ] } }, diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb index 6808de9..b272735 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb @@ -84,29 +84,7 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "expiry_str: '1784802224' 1784801151.7235122\n" - ] - }, - { - "ename": "ValueError", - "evalue": "invalid literal for int() with base 10: \"'1784802224'\"", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m python_interface_to_workflows.submit_to_argo \u001b[38;5;28;01mimport\u001b[39;00m submit_workflow_to_argo\n\u001b[32m 2\u001b[39m \n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m submit_workflow_to_argo(w)\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/submit_to_argo.py:13\u001b[39m, in \u001b[36msubmit_workflow_to_argo\u001b[39m\u001b[34m(w)\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msubmit_workflow_to_argo\u001b[39m(w: Workflow):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[30;43mset_token_env_variable\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstaging\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43;01mTrue\u001b[39;49;00m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 14\u001b[39m dotenv.load_dotenv(dotenv_path=\u001b[33m\"\u001b[39m\u001b[33msrc/.env\u001b[39m\u001b[33m\"\u001b[39m, override=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 15\u001b[39m w.namespace = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mNAMESPACE\u001b[39m\u001b[33m\"\u001b[39m)\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/keycloak_checker.py:39\u001b[39m, in \u001b[36mset_token_env_variable\u001b[39m\u001b[34m(staging)\u001b[39m\n\u001b[32m 31\u001b[39m code_challenge, code_challenge_method = generate_code_challenge(code_verifier)\n\u001b[32m 32\u001b[39m auth_url = keycloak_openid.auth_url(\n\u001b[32m 33\u001b[39m redirect_uri=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mport\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 34\u001b[39m scope=\u001b[33m\"\u001b[39m\u001b[33mopenid posix-uid profile email fedid\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 37\u001b[39m code_challenge_method=code_challenge_method,\n\u001b[32m 38\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mmatch\u001b[39;00m \u001b[30;43mopen_auth_url\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mauth_url\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mport\u001b[39;49m\u001b[30;43m)\u001b[39;49m:\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mcase\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 41\u001b[39m token: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = ( \u001b[38;5;66;03m# pyright: ignore[reportUnknownVariableType]\u001b[39;00m\n\u001b[32m 42\u001b[39m keycloak_openid.token( \u001b[38;5;66;03m# pyright: ignore[reportUnknownMemberType]\u001b[39;00m\n\u001b[32m 43\u001b[39m grant_type=\u001b[33m\"\u001b[39m\u001b[33mauthorization_code\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 47\u001b[39m )\n\u001b[32m 48\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/open_auth_url.py:36\u001b[39m, in \u001b[36mopen_auth_url\u001b[39m\u001b[34m(auth_url, port)\u001b[39m\n\u001b[32m 34\u001b[39m expiry_str: \u001b[38;5;28mstr\u001b[39m = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mEXPIRY\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;66;03m# pyright: ignore\u001b[39;00m\n\u001b[32m 35\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mexpiry_str: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpiry_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m, \u001b[38;5;28mstr\u001b[39m(time.time()))\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (expiry_str == \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[30;43mint\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mexpiry_str\u001b[39;49m\u001b[30;43m)\u001b[39;49m) <= \u001b[38;5;28mfloat\u001b[39m(time.time()):\n\u001b[32m 37\u001b[39m httpd = _ReusingHTTPServer((\u001b[33m\"\u001b[39m\u001b[33mlocalhost\u001b[39m\u001b[33m\"\u001b[39m, port), CallbackHandler)\n\u001b[32m 38\u001b[39m webbrowser.open(auth_url)\n", - "\u001b[31mValueError\u001b[39m: invalid literal for int() with base 10: \"'1784802224'\"" - ] - } - ], + "outputs": [], "source": [ "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", "\n", @@ -130,7 +108,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.15" + "version": "3.11.-1" } }, "nbformat": 4, diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb index fde4ccd..56cc504 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb @@ -219,29 +219,7 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "expiry_str: '1784802224' 1784801151.7235122\n" - ] - }, - { - "ename": "ValueError", - "evalue": "invalid literal for int() with base 10: \"'1784802224'\"", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m python_interface_to_workflows.submit_to_argo \u001b[38;5;28;01mimport\u001b[39;00m submit_workflow_to_argo\n\u001b[32m 2\u001b[39m \n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m submit_workflow_to_argo(w)\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/submit_to_argo.py:13\u001b[39m, in \u001b[36msubmit_workflow_to_argo\u001b[39m\u001b[34m(w)\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msubmit_workflow_to_argo\u001b[39m(w: Workflow):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[30;43mset_token_env_variable\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstaging\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43;01mTrue\u001b[39;49;00m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 14\u001b[39m dotenv.load_dotenv(dotenv_path=\u001b[33m\"\u001b[39m\u001b[33msrc/.env\u001b[39m\u001b[33m\"\u001b[39m, override=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 15\u001b[39m w.namespace = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mNAMESPACE\u001b[39m\u001b[33m\"\u001b[39m)\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/keycloak_checker.py:39\u001b[39m, in \u001b[36mset_token_env_variable\u001b[39m\u001b[34m(staging)\u001b[39m\n\u001b[32m 31\u001b[39m code_challenge, code_challenge_method = generate_code_challenge(code_verifier)\n\u001b[32m 32\u001b[39m auth_url = keycloak_openid.auth_url(\n\u001b[32m 33\u001b[39m redirect_uri=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mport\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 34\u001b[39m scope=\u001b[33m\"\u001b[39m\u001b[33mopenid posix-uid profile email fedid\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 37\u001b[39m code_challenge_method=code_challenge_method,\n\u001b[32m 38\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mmatch\u001b[39;00m \u001b[30;43mopen_auth_url\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mauth_url\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mport\u001b[39;49m\u001b[30;43m)\u001b[39;49m:\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mcase\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 41\u001b[39m token: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = ( \u001b[38;5;66;03m# pyright: ignore[reportUnknownVariableType]\u001b[39;00m\n\u001b[32m 42\u001b[39m keycloak_openid.token( \u001b[38;5;66;03m# pyright: ignore[reportUnknownMemberType]\u001b[39;00m\n\u001b[32m 43\u001b[39m grant_type=\u001b[33m\"\u001b[39m\u001b[33mauthorization_code\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 47\u001b[39m )\n\u001b[32m 48\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/open_auth_url.py:36\u001b[39m, in \u001b[36mopen_auth_url\u001b[39m\u001b[34m(auth_url, port)\u001b[39m\n\u001b[32m 34\u001b[39m expiry_str: \u001b[38;5;28mstr\u001b[39m = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mEXPIRY\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;66;03m# pyright: ignore\u001b[39;00m\n\u001b[32m 35\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mexpiry_str: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpiry_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m, \u001b[38;5;28mstr\u001b[39m(time.time()))\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (expiry_str == \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[30;43mint\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mexpiry_str\u001b[39;49m\u001b[30;43m)\u001b[39;49m) <= \u001b[38;5;28mfloat\u001b[39m(time.time()):\n\u001b[32m 37\u001b[39m httpd = _ReusingHTTPServer((\u001b[33m\"\u001b[39m\u001b[33mlocalhost\u001b[39m\u001b[33m\"\u001b[39m, port), CallbackHandler)\n\u001b[32m 38\u001b[39m webbrowser.open(auth_url)\n", - "\u001b[31mValueError\u001b[39m: invalid literal for int() with base 10: \"'1784802224'\"" - ] - } - ], + "outputs": [], "source": [ "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", "\n", From 6594a7b3cf63f4e8f509b504cc08173b2d9d3ec4 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Tue, 28 Jul 2026 10:42:03 +0000 Subject: [PATCH 4/5] feat(ipynb): updates copier and refactors notebooks --- .devcontainer/devcontainer.json | 11 ++++++++-- scripts/makecopiercorrect.sh | 20 +++++++++++++++++++ scripts/runthefiles.sh | 2 +- .../.devcontainer/devcontainer.json | 11 ++++++++-- .../scripts/runthefiles.sh.jinja | 6 +++--- .../submit_workflow.py.jinja | 2 +- .../notebook_division.ipynb.jinja | 16 ++++++++------- .../notebook_example.ipynb.jinja | 12 +++++------ .../tests/test_keycloak_checker.py.jinja | 2 +- .../tests/test_submit_to_graphql.py.jinja | 2 +- .../{divisionyaml.yaml => divisionyaml.txt} | 0 .../create_division_yaml.py | 2 +- .../{ => notebooks}/notebook_division.ipynb | 8 ++++---- .../{ => notebooks}/notebook_example.ipynb | 6 +++--- 14 files changed, 68 insertions(+), 32 deletions(-) rename src/copier_template/src/{{ project_name }}/workflow_definitions/{ => notebooks}/notebook_division.ipynb.jinja (87%) rename src/copier_template/src/{{ project_name }}/workflow_definitions/{ => notebooks}/notebook_example.ipynb.jinja (96%) rename src/python_interface_to_workflows/templates/{divisionyaml.yaml => divisionyaml.txt} (100%) rename src/python_interface_to_workflows/workflow_definitions/{ => notebooks}/notebook_division.ipynb (93%) rename src/python_interface_to_workflows/workflow_definitions/{ => notebooks}/notebook_example.ipynb (97%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2fb7099..cfa6f9e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -40,8 +40,15 @@ "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", "ms-azuretools.vscode-docker", - "ms-toolsai.jupyter-keymap", - "ms-toolsai.jupyter" + "ms-python.debugpy", + "ms-python.vscode-python-envs", + "ms-python.vscode-pylance", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.jupyter", + "ms-azuretools.vscode-containers", + "ms-toolsai.jupyter-keymap" ] } }, diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 973bd00..55b5d63 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -6,6 +6,7 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/* do + [[ -d "$file" ]] && continue [[ $file == *.jinja ]] && continue sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" @@ -21,9 +22,28 @@ do mv "$file" "$file.jinja" done +for file in "../../copier_template/src/{{ project_name }}/workflow_definitions/notebooks"/* +do + [[ -d "$file" ]] && continue + [[ $file == *.jinja ]] && continue + + sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" + sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + + sed -i '1i{% raw %}' "$file" + echo '{% endraw %}' >> "$file" + sed -i \ + -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ + -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + "$file" + + + mv "$file" "$file.jinja" +done for file in "../../copier_template/src/{{ project_name }}/templates"/* do + [[ -d "$file" ]] && continue [[ $file == *.jinja ]] && continue sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index 960a7d8..662bf8e 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -2,8 +2,8 @@ cd src/python_interface_to_workflows/workflow_definitions for file in * do +[[ -d "$file" ]] && continue uv run "$file" done -mv *.yaml ../templates/ mv *.txt ../templates/ git add -u ../templates/ diff --git a/src/copier_template/.devcontainer/devcontainer.json b/src/copier_template/.devcontainer/devcontainer.json index 2fb7099..cfa6f9e 100644 --- a/src/copier_template/.devcontainer/devcontainer.json +++ b/src/copier_template/.devcontainer/devcontainer.json @@ -40,8 +40,15 @@ "ryanluker.vscode-coverage-gutters", "charliermarsh.ruff", "ms-azuretools.vscode-docker", - "ms-toolsai.jupyter-keymap", - "ms-toolsai.jupyter" + "ms-python.debugpy", + "ms-python.vscode-python-envs", + "ms-python.vscode-pylance", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.jupyter", + "ms-azuretools.vscode-containers", + "ms-toolsai.jupyter-keymap" ] } }, diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index f324322..c4d14a7 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -2,8 +2,8 @@ cd src/{{project_name}}/workflow_definitions for file in * do +[[ -d "$file" ]] && continue uv run "$file" done -mv *.yaml ../templates/ -mv *.txt ../templates/ -git add -u ../templates/ +mv *.txt src/{{project_name}}/templates/ +git add -u src/{{project_name}}/templates/ diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index 597599a..a2dfed9 100644 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -12,7 +12,7 @@ from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_toke def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) - token: str = set_token_env_variable(True) + token: str = set_token_env_variable() host: str = os.environ.get("HOST") # pyright:ignore visit: str = os.environ.get("VISIT") # pyright:ignore diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja similarity index 87% rename from src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja rename to src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja index c8b4218..c7c8465 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja @@ -1,4 +1,5 @@ -{% raw %}{ +{% raw %} +{ "cells": [ { "cell_type": "markdown", @@ -7,7 +8,7 @@ "source": [ "# Running in VSCode:\n", "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", "2. Hit F1, run Jupyter: Import Notebook to Script\n", "3. Click notebook_division.ipynb\n", "4. Run the cells sequentially" @@ -59,7 +60,7 @@ " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", " the remainder, output float, and output string to a json file\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", " },\n", " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", ") as w:\n", @@ -75,7 +76,7 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n", + "with open(\"division_from_jupyter.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, @@ -84,16 +85,17 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, + "outputs": [], "source": [ - "from {% endraw %}{{project_name}}{% raw %}.submit_to_argo import submit_workflow_to_argo\n", + "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", - "submit_workflow_to_argo(w)" + "submit_workflow(w)" ] } ], "metadata": { "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", + "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", "language": "python", "name": "python3" }, diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja similarity index 96% rename from src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja rename to src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja index aca499c..0f829f6 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_example.ipynb.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_example.ipynb.jinja @@ -7,7 +7,7 @@ "source": [ "# Running in VSCode:\n", "\n", - "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", "2. Hit F1, run Jupyter: Import Notebook to Script\n", "3. Click notebook_example.ipynb\n", "4. Run the cells sequentially" @@ -175,7 +175,7 @@ " \"workflows.argoproj.io/title\": \"example remade via hera\",\n", " \"workflows.argoproj.io/description\": \"\"\"Replicates the functionality of\n", "example.yaml\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", " },\n", " volumes=Volume(name=\"tmpdir\", mount_path=\"/tmp/\", size=\"1Gi\"),\n", ") as w:\n", @@ -219,16 +219,16 @@ "execution_count": null, "id": "7e9f88cb", "metadata": {}, - "source": [ - "from {% endraw %}{{project_name}}{% raw %}.submit_to_argo import submit_workflow_to_argo\n", + "source": [ + "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", "\n", - "submit_workflow_to_argo(w)" + "submit_workflow(w)" ] } ], "metadata": { "kernelspec": { - "display_name": "python-interface-to-workflows (3.11.x)", + "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", "language": "python", "name": "python3" }, diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index 4b27f07..b960007 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -45,7 +45,7 @@ def test_set_token_env_variable( keycloak.token.return_value = token keycloak.refresh_token.return_value = token keycloak.decode_token.return_value = {"exp": 123456789} - assert set_token_env_variable(staging) == "fake_token" + assert set_token_env_variable() == "fake_token" mock_token_expired.assert_called_once_with("https://mock.site", port) if return_present: diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 25add52..265d189 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from {{project_name}}.submit_workflow import submit_workflow diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.txt similarity index 100% rename from src/python_interface_to_workflows/templates/divisionyaml.yaml rename to src/python_interface_to_workflows/templates/divisionyaml.txt diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py index 8a57ae1..327618f 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py @@ -45,5 +45,5 @@ def do_division(a: int, b: int): do_division(name="first", arguments={"a": 2, "b": 5}) -with open("divisionyaml.yaml", "w") as div: +with open("divisionyaml.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb similarity index 93% rename from src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb rename to src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb index b272735..9113672 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_division.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "494174ef", "metadata": {}, "outputs": [], @@ -75,7 +75,7 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n", + "with open(\"../../templates/division_from_jupyter.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, @@ -86,9 +86,9 @@ "metadata": {}, "outputs": [], "source": [ - "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", + "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow_to_argo(w)" + "submit_workflow(w)" ] } ], diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb similarity index 97% rename from src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb rename to src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb index 56cc504..c1018ea 100644 --- a/src/python_interface_to_workflows/workflow_definitions/notebook_example.ipynb +++ b/src/python_interface_to_workflows/workflow_definitions/notebooks/notebook_example.ipynb @@ -210,7 +210,7 @@ "outputs": [], "source": [ "\n", - "with open(\"example.txt\", \"w\") as div:\n", + "with open(\"../../templates/example_from_jupyter.txt\", \"w\") as div:\n", " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" ] }, @@ -221,9 +221,9 @@ "metadata": {}, "outputs": [], "source": [ - "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", + "from python_interface_to_workflows.submit_workflow import submit_workflow\n", "\n", - "submit_workflow_to_argo(w)" + "submit_workflow(w)" ] } ], From 7478b81b56a269b51c84ca0e8aad44db347a3bbc Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 10:59:53 +0000 Subject: [PATCH 5/5] docs(copier): removes division notebook template --- .../notebooks/notebook_division.ipynb.jinja | 118 ------------------ 1 file changed, 118 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja deleted file mode 100644 index c7c8465..0000000 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/notebooks/notebook_division.ipynb.jinja +++ /dev/null @@ -1,118 +0,0 @@ -{% raw %} -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e3146f1", - "metadata": {}, - "source": [ - "# Running in VSCode:\n", - "\n", - "1. Set kernal to {% endraw %}{{repo_name}}{% raw %} 3.11.x\n", - "2. Hit F1, run Jupyter: Import Notebook to Script\n", - "3. Click notebook_division.ipynb\n", - "4. Run the cells sequentially" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "494174ef", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "from hera.workflows import (\n", - " Artifact,\n", - " EmptyDirVolume,\n", - " Steps,\n", - " Workflow,\n", - " script, # pyright: ignore[reportUnknownVariableType]\n", - ")\n", - "from hera.workflows import models as m\n", - "\n", - "\n", - "@script(\n", - " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", - " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", - ")\n", - "def do_division(a: int, b: int):\n", - " div = a / b\n", - " intdiv = a // b\n", - " remain = a % b\n", - " dictionary_of_results = {\n", - " \"divide\": div,\n", - " \"quotient\": intdiv,\n", - " \"remainder\": remain,\n", - " }\n", - " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", - " json.dump(dictionary_of_results, otpt)\n", - "\n", - "\n", - "with Workflow(\n", - " generate_name=\"hera-division-\", # when running on graphql this should be name\n", - " entrypoint=\"divide\",\n", - " api_version=\"argoproj.io/v1alpha1\",\n", - " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", - " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", - " annotations={\n", - " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", - " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", - " the remainder, output float, and output string to a json file\"\"\",\n", - " \"workflows.diamond.ac.uk/repository\": \"https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}\",\n", - " },\n", - " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", - ") as w:\n", - " with Steps(name=\"divide\"):\n", - " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04708f46", - "metadata": {}, - "outputs": [], - "source": [ - "with open(\"division_from_jupyter.txt\", \"w\") as div:\n", - " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e9f88cb", - "metadata": {}, - "outputs": [], - "source": [ - "from {% endraw %}{{project_name}}{% raw %}.submit_workflow import submit_workflow\n", - "\n", - "submit_workflow(w)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "{% endraw %}{{repo_name}}{% raw %} (3.11.x)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.15" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} -{% endraw %}