Allow creating EOAP containers without xcube, and vice versa

CHANGES.md

@@ -4,6 +4,8 @@
   environment file paths (#42)
 * Add `s:version` metadata field to CWL as a synonym for
   `s:softwareVersion` (#90)
+* Allow creation of EOAP-only and xcube-server-only images, omitting
+  unnecessary dependencies (#56)
 
 ## Changes in 0.1.2
 

docs/xcetool.md

@@ -40,6 +40,16 @@ Options:
     timestamp-based tag will be generated automatically.
 -   `-a`, `--eoap` `PATH`: Write a CWL file defining an Earth Observation
     Application Package to the specified path.
+-   `-s`, `--skip-build`: Prepare the Dockerfile and build context, but
+    don't actually build the image. Mainly useful in conjunction with the
+    `--build-dir` option, so that the generated build configuration can
+    used by an external tool or examined.
+-   `-n`, `--no-eoap`: Do not add EOAP functionality to the built image.
+    The image will only be useable in xcube server mode. This option can
+    help to reduce the size of the image.
+-   `-x`, `--no-xcube`: Do not add xcube server/viewer functionality to
+    the built image. The image will only be useable in EOAP mode. This
+    option can help to reduce the size of the image.
 -   `--help`: Show a help message for this subcommand and exit.
 
 ### `xcetool image run`

test/test_cli.py

@@ -66,9 +66,19 @@ def test_make_script(
 @pytest.mark.parametrize("specify_dir", [False, True])
 @pytest.mark.parametrize("specify_env", [False, True])
 @pytest.mark.parametrize("specify_eoap", [False, True])
+@pytest.mark.parametrize("skip_build", [False, True])
+@pytest.mark.parametrize("no_eoap", [False, True])
+@pytest.mark.parametrize("no_xcube", [False, True])
 @patch("xcengine.cli.ImageBuilder")
 def test_image_build(
-    builder_mock, tmp_path, specify_dir, specify_env, specify_eoap
+    builder_mock,
+    tmp_path,
+    specify_dir,
+    specify_env,
+    specify_eoap,
+    skip_build,
+    no_eoap,
+    no_xcube,
 ):
     (nb_path := tmp_path / "foo.ipynb").touch()
     (env_path := tmp_path / "environment.yml").touch()
@@ -85,6 +95,9 @@ def test_image_build(
         + (["--build-dir", str(build_dir)] if specify_dir else [])
         + (["--environment", str(env_path)] if specify_env else [])
         + (["--eoap", str(eoap_path)] if specify_eoap else [])
+        + (["--skip-build"] if skip_build else [])
+        + (["--no-eoap"] if no_eoap else [])
+        + (["--no-xcube"] if no_xcube else [])
         + [str(nb_path)],
     )
     assert result.output.startswith("Built image")
@@ -95,7 +108,9 @@ def test_image_build(
         tag=tag,
         build_dir=(build_dir if specify_dir else ANY),
     )
-    instance_mock.build.assert_called_once_with(skip_build=False)
+    instance_mock.build.assert_called_once_with(
+        skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube
+    )
     if specify_eoap:
         assert yaml.safe_load(eoap_path.read_text()) == cwl
 

test/test_core.py

@@ -354,7 +354,9 @@ def test_script_creator_notebook_config():
 def test_script_creator_notebook_config_http(httpserver):
     http_path = "/mynotebook.ipynb"
     nb_path = pathlib.Path(__file__).parent / "data" / "paramtest.ipynb"
-    httpserver.expect_request(http_path).respond_with_data(nb_path.read_bytes())
+    httpserver.expect_request(http_path).respond_with_data(
+        nb_path.read_bytes()
+    )
     script_creator = ScriptCreator(httpserver.url_for(http_path))
     config = script_creator.nb_params.config
     assert config["environment_file"] == "my-environment.yml"
@@ -382,12 +384,20 @@ def test_image_builder_write_dockerfile(tmp_path):
 @patch("docker.from_env")
 @pytest.mark.parametrize("env_type", ["none", "local", "http"])
 @pytest.mark.parametrize("skip_build", [False, True])
+@pytest.mark.parametrize("with_eoap", [False, True])
+@pytest.mark.parametrize("with_xcube", [False, True])
+@pytest.mark.parametrize("pystac_in_deps", [False, True])
+@pytest.mark.parametrize("xcube_in_deps", [False, True])
 def test_image_builder_build_dir(
-        from_env_mock,
-        tmp_path,
-        httpserver,
-        env_type,
-        skip_build
+    from_env_mock,
+    tmp_path,
+    httpserver,
+    env_type,
+    skip_build,
+    with_eoap,
+    with_xcube,
+    pystac_in_deps,
+    xcube_in_deps,
 ):
     client_mock = Mock(docker.client.DockerClient)
     client_mock.images.build.return_value = None, None
@@ -398,16 +408,22 @@ def test_image_builder_build_dir(
     env_def = {
         "name": "foo",
         "channels": "bar",
-        "dependencies": ["python >=3.13", "baz >=42.0"],
+        "dependencies": ["python >=3.13", "baz >=42.0"]
+        + (["pystac"] if pystac_in_deps else [])
+        + (["xcube"] if xcube_in_deps else []),
     }
     build_env_path.write_text(yaml.safe_dump(env_def))
     env_http = "/env2.yaml"
 
     match env_type:
-        case "none": env_param = None
-        case "local": env_param = build_env_path
+        case "none":
+            env_param = None
+        case "local":
+            env_param = build_env_path
         case "http":
-            httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes())
+            httpserver.expect_request(env_http).respond_with_data(
+                build_env_path.read_bytes()
+            )
             env_param = httpserver.url_for(env_http)
         case _:
             raise RuntimeError(f"Unknown env type {env_type}")
@@ -418,7 +434,9 @@ def test_image_builder_build_dir(
         build_dir,
         None,
     )
-    image_builder.build(skip_build=skip_build)
+    image_builder.build(
+        skip_build=skip_build, with_eoap=with_eoap, with_xcube=with_xcube
+    )
     if skip_build:
         from_env_mock.assert_not_called()
     else:
@@ -430,7 +448,12 @@ def test_image_builder_build_dir(
     if env_type != "none":
         assert output_env["name"] == env_def["name"]
         assert output_env["channels"] == env_def["channels"]
-        assert set(output_env["dependencies"]) >= set(env_def["dependencies"])
+
+        output_deps = set(output_env["dependencies"])
+        input_deps = set(env_def["dependencies"])
+        assert output_deps >= input_deps
+        assert ("pystac" in output_deps) == pystac_in_deps or with_eoap
+        assert ("xcube" in output_deps) == xcube_in_deps or with_xcube
 
     cwl = image_builder.create_cwl()
     assert "cwlVersion" in cwl

test/test_util.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 import os
 import sys
@@ -85,6 +86,67 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog):
         }
 
 
+def test_write_stac_no_pystac(tmp_path, dataset):
+    # Import hooks are the recommended "clean" way to do this, but don't work
+    # in this case.
+    orig_import = __import__
+
+    def import_mock(name, *args):
+        if name == "pystac":
+            raise ModuleNotFoundError("No module named 'pystac'")
+        return orig_import(name, *args)
+
+    with mock.patch("builtins.__import__", side_effect=import_mock):
+        # pytest imports xcengine.util long before we can patch __import__,
+        # so we delete pystac from util's namespace (if present) instead.
+        # This gives a NameError on access rather than a ModuleNotFoundError
+        # on import, but the important thing is to break any implementation
+        # that tries to import pystac without checking if it's available.
+        import xcengine.util
+
+        xcengine.util.__dict__.pop("pystac", None)
+        from xcengine.util import write_stac
+
+        write_stac({"ds1": dataset}, tmp_path)
+        # We want nothing to happen here, so no explicit assertions.
+
+
+def test_start_server_no_xcube(dataset):
+    import io
+
+    orig_import = __import__
+
+    def import_mock(name, *args):
+        if name == "xcube" or name.startswith("xcube."):
+            raise ModuleNotFoundError(f"No module named {name}")
+        return orig_import(name, *args)
+
+    with mock.patch("builtins.__import__", side_effect=import_mock):
+        import xcengine.util
+
+        util_vars = (
+            k
+            for k in xcengine.util.__dict__.keys()
+            if k == "xcube" or k.startswith("xcube.")
+        )
+        for v in util_vars:
+            del xcengine.util.__dict__[v]
+        from xcengine.util import start_server
+
+        logger = logging.getLogger("test-start-server-logger")
+        logger.setLevel(logging.INFO)
+        logger.addHandler(logging.StreamHandler(log_stream := io.StringIO()))
+        start_server(
+            {"ds1": dataset},
+            {},
+            argparse.Namespace(batch=False, from_saved=False),
+            logger,
+        )
+        logged = log_stream.getvalue()
+        assert "Not starting" in logged
+        assert "Starting server" not in logged
+
+
 @pytest.mark.parametrize("eoap_mode", [False, True])
 @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"])
 def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format):

xcengine/cli.py

@@ -151,7 +151,19 @@ def image_cli():
     "--skip-build",
     is_flag=True,
     help="Prepare the Dockerfile and build context, but don't actually build "
-    "the image"
+    "the image",
+)
+@click.option(
+    "-n",
+    "--no-eoap",
+    is_flag=True,
+    help="Do not add EOAP functionality to built image",
+)
+@click.option(
+    "-x",
+    "--no-xcube",
+    is_flag=True,
+    help="Do not add xcube server/viewer functionality to built image",
 )
 @notebook_argument
 def build(
@@ -160,27 +172,37 @@ def build(
     environment: pathlib.Path,
     tag: str,
     eoap: pathlib.Path,
-    skip_build: bool
+    skip_build: bool,
+    no_eoap: bool,
+    no_xcube: bool,
 ) -> None:
     if environment is None:
         LOGGER.info("No environment file specified on command line.")
 
     class InitArgs(TypedDict):
         notebook: pathlib.Path
-        environment: pathlib.Path
+        environment: pathlib.Path | None
         tag: str
 
+    class BuildArgs(TypedDict):
+        skip_build: bool
+        with_eoap: bool
+        with_xcube: bool
+
     init_args = InitArgs(notebook=notebook, environment=environment, tag=tag)
+    build_args = BuildArgs(
+        skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube
+    )
     if build_dir:
         image_builder = ImageBuilder(build_dir=build_dir, **init_args)
         os.makedirs(build_dir, exist_ok=True)
-        image = image_builder.build(skip_build=skip_build)
+        image = image_builder.build(**build_args)
     else:
         with tempfile.TemporaryDirectory() as temp_dir:
             image_builder = ImageBuilder(
                 build_dir=pathlib.Path(temp_dir), **init_args
             )
-            image = image_builder.build(skip_build=skip_build)
+            image = image_builder.build(**build_args)
     if eoap:
 
         class IndentDumper(yaml.Dumper):
@@ -196,8 +218,8 @@ def increase_indent(self, flow=False, indentless=False):
         )
     print(
         f"Built image with tags {image.tags}"
-        if image is not None else
-        "No image built"
+        if image is not None
+        else "No image built"
     )
 
 

xcengine/core.py

@@ -242,6 +242,8 @@ def __init__(
     def build(
         self,
         skip_build: bool = False,
+        with_eoap: bool = True,
+        with_xcube: bool = True,
     ) -> Image | None:
         self.script_creator.convert_notebook_to_script(self.build_dir)
         if self.environment:
@@ -255,7 +257,11 @@ def build(
             )
             env_def = self.export_conda_env()
         # We need xcube for server/viewer and pystac for EOAP stage-in/out
-        self.add_packages_to_environment(env_def, ["xcube", "pystac"])
+        self.add_packages_to_environment(
+            env_def,
+            (["xcube"] if with_xcube else [])
+            + (["pystac"] if with_eoap else []),
+        )
         with open(self.build_dir / "environment.yml", "w") as fh:
             fh.write(yaml.safe_dump(env_def))
         self.write_dockerfile(self.build_dir / "Dockerfile")

xcengine/parameters.py

@@ -4,7 +4,6 @@
 import typing
 from typing import Any, ClassVar
 
-import pystac
 import xarray as xr
 import yaml
 
@@ -177,6 +176,7 @@ def read_datasets_from_product(
                 f"Stage-in directory {stage_in_path} does not contain a "
                 f'"catalog.json" file.'
             )
+        import pystac
         catalog = pystac.Catalog.from_file(catalog_path)
         item_links = [link for link in catalog.links if link.rel == "item"]
         expected_names = set(self.dataset_inputs)
@@ -202,9 +202,10 @@ def read_datasets_from_product(
     @staticmethod
     def read_staged_in_dataset(
         stage_in_path: pathlib.Path,
-        catalog: pystac.Catalog,
+        catalog: "pystac.Catalog",
         param_name: str,
     ) -> xr.Dataset:
+        import pystac
         item_links = [link for link in catalog.links if link.rel == "item"]
         item = next(
             filter(