diff --git a/scripts/checkyamlcompliance.py b/scripts/checkyamlcompliance.py index 834655d..2f64a09 100644 --- a/scripts/checkyamlcompliance.py +++ b/scripts/checkyamlcompliance.py @@ -24,6 +24,8 @@ api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: + case "ClusterWorkflowTemplate": + clst_tmpt = True case "Workflow": clst_tmpt = True case "WorkflowTemplate": diff --git a/src/copier_template/scripts/checkyamlcompliance.py.jinja b/src/copier_template/scripts/checkyamlcompliance.py.jinja index 020b14a..b020cd6 100644 --- a/src/copier_template/scripts/checkyamlcompliance.py.jinja +++ b/src/copier_template/scripts/checkyamlcompliance.py.jinja @@ -24,6 +24,8 @@ for file in yamllist: api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: + case "ClusterWorkflowTemplate": + clst_tmpt = True case "Workflow": clst_tmpt = True case _: diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 816c35d..c09fef6 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -11,16 +11,16 @@ AUTH= 3. Build the dev container -# Submitting a workflow to Argo +# Submitting a workflow 1. At the start of your workflow definition file, add: ```python -from {{project_name}}.submit_to_argo import submit_workflow_to_argo +from {{project_name}}.submit_to_graphql import submit_workflow ``` 2. At the end of a workflow definition file, once your workflow (w) is created, add: ```python -submit_workflow_to_argo(w) +submit_workflow(w) ``` NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are diff --git a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja deleted file mode 100644 index 1b30683..0000000 --- a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja +++ /dev/null @@ -1,24 +0,0 @@ -import os -from typing import cast - -import dotenv -from hera.shared import global_config -from hera.workflows import Workflow, WorkflowsService -from hera.workflows import models as m - -from {{project_name}}.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow_to_argo(w: Workflow): - set_token_env_variable() - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - w.namespace = os.environ.get("VISIT") - w.workflows_service = WorkflowsService( - host=str(os.environ.get("HOST")).strip("'"), - token=str(os.environ.get("TOKEN")).strip("'"), - ) - global_config.image = str(os.environ.get("DEFAULT_IMAGE")) - submitted_w = cast(m.Workflow, w.create()) - name = submitted_w.metadata.name - namespace = submitted_w.metadata.namespace - print(f"submitted {name} to {namespace}") diff --git a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja deleted file mode 100644 index edfb8c9..0000000 --- a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja +++ /dev/null @@ -1,36 +0,0 @@ -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport -import os -from {{project_name}}.auth.keycloak_checker import set_token_env_variable -def submit_to_graphql(): - token = set_token_env_variable() - transport = AIOHTTPTransport( - url="https://workflows.diamond.ac.uk/graphql", - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation SubmitDivision { - submitWorkflowTemplate( - name: "division" - visit: { - proposalCode: "ks", - proposalNumber: 10000, - number: 3 - } - parameters: { - numinput: "19", - numdivisor: "10" - } - ){ - name -} -} -""") - result = client.execute(mutation) - visit = "ks10000-3/c" - name = str(result["submitWorkflowTemplate"]["name"]) - print(f"Job '{visit}-{name}' submitted.") diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja new file mode 100644 index 0000000..597599a --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -0,0 +1,50 @@ +{% raw %} +import os + +import dotenv +from gql import Client, gql +from gql.transport.aiohttp import AIOHTTPTransport +from hera.workflows import Workflow + +from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable + + +def submit_workflow(w: Workflow): + yamlstr = w.to_yaml() # pyright:ignore + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + token: str = set_token_env_variable(True) + host: str = os.environ.get("HOST") # pyright:ignore + visit: str = os.environ.get("VISIT") # pyright:ignore + + transport = AIOHTTPTransport( + url=host, + headers={"Authorization": f"Bearer {token}"}, + ) + client = Client( + transport=transport, + fetch_schema_from_transport=True, + ) + mutation = gql(""" +mutation Submit($visit: VisitInput!, $manifest: String!) { + submitWorkflow( + visit: $visit + manifest: $manifest + ) { + name + } +} +""") + result = client.execute( + mutation, + variable_values={ + "visit": { + "proposalCode": str(visit[:2]), + "proposalNumber": int(visit[2:7]), + "number": int(visit[-1]), + }, + "manifest": f"""{yamlstr}""", + }, + ) + name = str(result["submitWorkflow"]["name"]) + print(f"Job '{name}' submitted to {visit}") +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja index 387db1c..78cb885 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja @@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-example- + name: hera-example annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index 42175cf..f438048 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -145,7 +145,7 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-", # when running on graphql this should be name + name="hera-example", # when running on argo this should be generate_name: ...- entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index fcf376e..25add52 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,17 +1,27 @@ from unittest.mock import MagicMock, patch -from {{project_name}}.submit_to_graphql import submit_to_graphql +from {{project_name}}.submit_workflow import submit_workflow -@patch("{{project_name}}.submit_to_graphql.set_token_env_variable") -@patch("{{project_name}}.submit_to_graphql.Client") -def test_submit_to_graphql(mock_client: MagicMock, mock_key: MagicMock): - mock_key.return_value = "fake_token" +@patch("{{project_name}}.submit_workflow.os.environ.get") +@patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") +@patch("{{project_name}}.submit_workflow.Workflow") +@patch("{{project_name}}.submit_workflow.set_token_env_variable") +@patch("{{project_name}}.submit_workflow.Client") +def test_submit_workflow_to_graphql( + mock_client: MagicMock, + mock_key: MagicMock, + mock_workflow: MagicMock, + mock_load_env: MagicMock, + mock_os_get: MagicMock, +): + mock_instance = MagicMock() - mock_token.return_value = "token" + mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = { - "submitWorkflowTemplate": {"name": "workflow123"} - } - submit_to_graphql() + mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} + submit_workflow(mock_workflow) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() + mock_workflow.to_yaml.assert_called_once() + mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/src/copier_template/tests/test_submit_workflow_to_argo.jinja b/src/copier_template/tests/test_submit_workflow_to_argo.jinja deleted file mode 100644 index c6eef98..0000000 --- a/src/copier_template/tests/test_submit_workflow_to_argo.jinja +++ /dev/null @@ -1,20 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from {{project_name}}.submit_to_argo import submit_workflow_to_argo - - -@patch("{{project_name}}.submit_to_argo.Workflow") -@patch("{{project_name}}.submit_to_argo.os.environ.get") -@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") -@patch("{{project_name}}.submit_to_argo.set_token_env_variable") -def test_submit_workflow_to_argo( - mock_key: MagicMock, - mock_load_env: MagicMock, - mock_get_env_var: MagicMock, - mock_workflow: MagicMock, -): - mock_workflow = MagicMock() - submit_workflow_to_argo(mock_workflow) - mock_key.assert_called_once_with(staging=True) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) diff --git a/src/python_interface_to_workflows/submit_to_argo.py b/src/python_interface_to_workflows/submit_to_argo.py deleted file mode 100644 index 0ec1233..0000000 --- a/src/python_interface_to_workflows/submit_to_argo.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -from typing import cast - -import dotenv -from hera.shared import global_config -from hera.workflows import Workflow, WorkflowsService -from hera.workflows import models as m - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow_to_argo(w: Workflow): - set_token_env_variable(staging=True) - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - w.namespace = os.environ.get("VISIT") - w.workflows_service = WorkflowsService( - host=str(os.environ.get("HOST")).strip("'"), - token=str(os.environ.get("TOKEN")).strip("'"), - ) - global_config.image = str(os.environ.get("DEFAULT_IMAGE")) - submitted_w = cast(m.Workflow, w.create()) - name = submitted_w.metadata.name - namespace = submitted_w.metadata.namespace - print(f"submitted {name} to {namespace}") diff --git a/src/python_interface_to_workflows/submit_to_graphql.py b/src/python_interface_to_workflows/submit_to_graphql.py deleted file mode 100644 index e449fc9..0000000 --- a/src/python_interface_to_workflows/submit_to_graphql.py +++ /dev/null @@ -1,38 +0,0 @@ -from gql import Client, gql -from gql.transport.aiohttp import AIOHTTPTransport - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -def submit_to_graphql(): - token = set_token_env_variable(True) - transport = AIOHTTPTransport( - url="https://staging.workflows.diamond.ac.uk/graphql", - headers={"Authorization": f"Bearer {token}"}, - ) - client = Client( - transport=transport, - fetch_schema_from_transport=True, - ) - mutation = gql(""" -mutation SubmitDivision { - submitWorkflowTemplate( - name: "division" - visit: { - proposalCode: "ks", - proposalNumber: 10000, - number: 3 - } - parameters: { - numinput: "19", - numdivisor: "10" - } - ){ - name -} -} -""") - result = client.execute(mutation) - visit = "ks10000-3/c" - name = str(result["submitWorkflowTemplate"]["name"]) - print(f"Job '{visit}-{name}' submitted.") diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py new file mode 100644 index 0000000..da3422a --- /dev/null +++ b/src/python_interface_to_workflows/submit_workflow.py @@ -0,0 +1,48 @@ +import os + +import dotenv +from gql import Client, gql +from gql.transport.aiohttp import AIOHTTPTransport +from hera.workflows import Workflow + +from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable + + +def submit_workflow(w: Workflow): + yamlstr = w.to_yaml() # pyright:ignore + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + token: str = set_token_env_variable(True) + host: str = os.environ.get("HOST") # pyright:ignore + visit: str = os.environ.get("VISIT") # pyright:ignore + + transport = AIOHTTPTransport( + url=host, + headers={"Authorization": f"Bearer {token}"}, + ) + client = Client( + transport=transport, + fetch_schema_from_transport=True, + ) + mutation = gql(""" +mutation Submit($visit: VisitInput!, $manifest: String!) { + submitWorkflow( + visit: $visit + manifest: $manifest + ) { + name + } +} +""") + result = client.execute( + mutation, + variable_values={ + "visit": { + "proposalCode": str(visit[:2]), + "proposalNumber": int(visit[2:7]), + "number": int(visit[-1]), + }, + "manifest": f"""{yamlstr}""", + }, + ) + name = str(result["submitWorkflow"]["name"]) + print(f"Job '{name}' submitted to {visit}") diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index 79b810a..c9ca1ea 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-division- + name: hera-division annotations: workflows.argoproj.io/description: |- Takes a numerical input and returns diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt index 857fb41..1ad8e0e 100644 --- a/src/python_interface_to_workflows/templates/example.txt +++ b/src/python_interface_to_workflows/templates/example.txt @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-example- + name: hera-example annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py index 3f063de..8a57ae1 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py @@ -28,7 +28,7 @@ def do_division(a: int, b: int): with Workflow( - generate_name="hera-division-", # when running on graphql this should be name + name="hera-division", # when running on argo this should be generate_name: ...- entrypoint="divide", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 7b9de3c..013387a 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -144,7 +144,7 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-", # when running on graphql this should be name + name="hera-example", # when running on argo this should be generate_name: ...- entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/tests/test_submit_to_argo.py b/tests/test_submit_to_argo.py deleted file mode 100644 index b8be696..0000000 --- a/tests/test_submit_to_argo.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo - - -@patch("python_interface_to_workflows.submit_to_argo.Workflow") -@patch("python_interface_to_workflows.submit_to_argo.os.environ.get") -@patch("python_interface_to_workflows.submit_to_argo.dotenv.load_dotenv") -@patch("python_interface_to_workflows.submit_to_argo.set_token_env_variable") -def test_submit_workflow_to_argo( - mock_key: MagicMock, - mock_load_env: MagicMock, - mock_get_env_var: MagicMock, - mock_workflow: MagicMock, -): - mock_workflow = MagicMock() - submit_workflow_to_argo(mock_workflow) - mock_key.assert_called_once_with(staging=True) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index 37e6a11..1eb8020 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -1,17 +1,27 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch -from python_interface_to_workflows.submit_to_graphql import submit_to_graphql +from python_interface_to_workflows.submit_workflow import submit_workflow -@patch("python_interface_to_workflows.submit_to_graphql.set_token_env_variable") -@patch("python_interface_to_workflows.submit_to_graphql.Client") -def test_submit_to_graphql(mock_client: MagicMock, mock_key: MagicMock): - mock_key.return_value = "fake_token" +@patch("python_interface_to_workflows.submit_workflow.os.environ.get") +@patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") +@patch("python_interface_to_workflows.submit_workflow.Workflow") +@patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") +@patch("python_interface_to_workflows.submit_workflow.Client") +def test_submit_workflow_to_graphql( + mock_client: MagicMock, + mock_key: MagicMock, + mock_workflow: MagicMock, + mock_load_env: MagicMock, + mock_os_get: MagicMock, +): + mock_instance = MagicMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = { - "submitWorkflowTemplate": {"name": "workflow123"} - } - submit_to_graphql() + mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} + submit_workflow(mock_workflow) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() + mock_workflow.to_yaml.assert_called_once() + mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True)