From bc7a6b65ce4ad82dbcc346b2c0bff8b6386dfee0 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:42:37 +0200 Subject: [PATCH 01/12] add state_date argument to init command --- src/virtualship/cli/commands.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 41f4d519e..b5110fac3 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -10,6 +10,7 @@ EXPEDITION, get_example_expedition, mfp_to_yaml, + validate_start_date, ) @@ -26,7 +27,15 @@ 'Marine Facilities Planning tool (specifically the "Export Coordinates > DD" option). ' "User edits are required after initialisation.", ) -def init(path, from_mfp): +@click.option( + "--start-date", + type=click.DateTime(formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"]), + default=None, + callback=validate_start_date, + help="The departure/start date of the expedition (required when using --from-mfp). " + "Expected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00.", +) +def init(path, from_mfp, start_date): """ Initialize a directory for a new expedition, with an expedition.yaml file. @@ -46,7 +55,9 @@ def init(path, from_mfp): mfp_file = Path(from_mfp) # Generate expedition.yaml from the MPF file click.echo(f"Generating schedule from {mfp_file}...") - mfp_to_yaml(mfp_file, expedition) + mfp_to_yaml(mfp_file, start_date, expedition) + # TODO: this print needs to be updated + # TODO: how to handle the ports?! Should be conditional on this kind of 'waypoint' being present in the MFP file. click.echo( "\n⚠️ The generated schedule does not contain TIME values or INSTRUMENT selections. ⚠️" "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the schedule configuration, " From 9c14261328476672ba376acf2296904957de18ab Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:43:23 +0200 Subject: [PATCH 02/12] handle timedeltas from mfp export, .xlsx only now that MFP export is .xslx only --- src/virtualship/utils.py | 126 +++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 46 deletions(-) diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 30f3dffcb..9126c72d6 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Literal, TextIO +import click import copernicusmarine import numpy as np import parcels @@ -181,35 +182,56 @@ def _generic_load_yaml(data: str, model: BaseModel) -> BaseModel: return model.model_validate(yaml.safe_load(data)) -def load_coordinates(file_path): - """Loads coordinates from a file based on its extension.""" +def validate_start_date(ctx, param, value): + """Callback to enforce and validate --start-date when --from-mfp is used.""" + if ctx.params.get("from_mfp"): + if not value: + raise click.BadParameter( + "The '--start-date' option is required when using '--from-mfp'." + "\n\nExpected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00." + ) + return value + + +def _load_mfpexport(file_path): if not os.path.isfile(file_path): raise FileNotFoundError(f"File not found: {file_path}") - ext = os.path.splitext(file_path)[-1].lower() - try: - if ext in [".xls", ".xlsx"]: - return pd.read_excel(file_path) - - if ext == ".csv": - return pd.read_csv(file_path) - - raise ValueError(f"Unsupported file extension {ext}.") + df = pd.read_excel(file_path) + return df.dropna(how="all", axis=1) # drop empty columns except Exception as e: raise RuntimeError( "Could not read coordinates data from the provided file. " - "Ensure it is either a csv or excel file." + "Ensure it is an exported .xlsx file from MFP." ) from e -def validate_coordinates(coordinates_data): - # Expected column headers - expected_columns = {"Station Type", "Name", "Latitude", "Longitude"} +def _validate_mfpdata(file_path): + """Load and validate MFP CruiseData export.""" + mfp_data = _load_mfpexport(file_path) + + # clean up column names + mfp_data.columns = mfp_data.columns.astype(str).str.strip() + mfp_data = mfp_data.loc[ + :, ~mfp_data.columns.str.startswith("Unnamed") & (mfp_data.columns != "") + ] - # Check if the headers match the expected ones - actual_columns = set(coordinates_data.columns) + expected_columns = { + "Station", + "Type", + "Latitude", + "Longitude", + "Sea Depth", + "Time at Station", + "Travel Time to Next", + "Distance to Next (NM)", + "Ship Speed (kn)", + "EEZ", + } + + actual_columns = set(mfp_data.columns) missing_columns = expected_columns - actual_columns if missing_columns: @@ -228,42 +250,39 @@ def validate_coordinates(coordinates_data): stacklevel=2, ) - # Drop unexpected columns (optional, only if you want to ensure strict conformity) - coordinates_data = coordinates_data[list(expected_columns)] - - # Continue with the rest of the function after validation... - coordinates_data = coordinates_data.dropna() + # Drop unexpected columns + mfp_data = mfp_data[list(expected_columns)] # Convert latitude and longitude to floats, replacing commas with dots # Handles case when the latitude and longitude have decimals with commas - if coordinates_data["Latitude"].dtype in ["object", "string"]: - coordinates_data["Latitude"] = coordinates_data["Latitude"].apply( + if mfp_data["Latitude"].dtype in ["object", "string"]: + mfp_data["Latitude"] = mfp_data["Latitude"].apply( lambda x: float(x.replace(",", ".")) ) - if coordinates_data["Longitude"].dtype in ["object", "string"]: - coordinates_data["Longitude"] = coordinates_data["Longitude"].apply( + if mfp_data["Longitude"].dtype in ["object", "string"]: + mfp_data["Longitude"] = mfp_data["Longitude"].apply( lambda x: float(x.replace(",", ".")) ) - return coordinates_data - + # convert 'Travel Time to Next' and 'Time at Station' to timedelta + mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( + lambda x: _mfp_string_to_timedelta(x) + ) + mfp_data["Time at Station"] = mfp_data["Time at Station"].apply( + lambda x: _mfp_string_to_timedelta(x) + ) -def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D417 - """ - Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version. + # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column + mfp_data["Total Time"] = ( + mfp_data["Travel Time to Next"] + mfp_data["Time at Station"] + ) - Parameters - ---------- - - excel_file_path (str): Path to the Excel file containing coordinate and instrument data. + return mfp_data - The function: - 1. Reads instrument and location data from the Excel file. - 2. Determines the maximum depth and buffer based on the instruments present. - 3. Ensures longitude and latitude values remain valid after applying buffer adjustments. - 4. returns the yaml information. - """ +def mfp_to_yaml(file_path: str, start_date: str, output_path: str): + """Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version.""" # avoid circular imports from virtualship.models import ( Expedition, @@ -274,19 +293,24 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 ) # Read data from file - coordinates_data = load_coordinates(coordinates_file_path) - - coordinates_data = validate_coordinates(coordinates_data) + mfp_data = _validate_mfpdata(file_path) # Generate waypoints waypoints = [] - for _, row in coordinates_data.iterrows(): + current_time, previous_timedelta = start_date, None + for i, row in mfp_data.iterrows(): + if i > 0: + current_time += previous_timedelta waypoints.append( Waypoint( - instrument=None, # instruments blank, to be built by user using `virtualship plan` UI or by interacting directly with YAML files + instrument=None, location=Location(latitude=row["Latitude"], longitude=row["Longitude"]), + time=current_time, ) ) + previous_timedelta = row[ + "Total Time" + ] # store total timedelta for next iteration # Create Schedule object schedule = Schedule( @@ -309,7 +333,17 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 ) # Save to YAML file - expedition.to_yaml(yaml_output_path) + expedition.to_yaml(output_path) + + +def _mfp_string_to_timedelta(value: str) -> timedelta: + """Handle MFP export string format (e.g., "0d 13h 13m").""" + if pd.isna(value): # last waypoint has no travel time to next, so will be NaN + return timedelta(0) + + value = value.replace("d", ":").replace("h", ":").replace("m", "") + days, hours, minutes = map(int, value.split(":")) + return timedelta(days=days, hours=hours, minutes=minutes) def _validate_numeric_to_timedelta( From 2c303d126243f8212e8a7ecfc69139a271411618 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:54:07 +0200 Subject: [PATCH 03/12] update click.echo(); no time instructions required --- src/virtualship/cli/commands.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index b5110fac3..2ecf71255 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -56,14 +56,16 @@ def init(path, from_mfp, start_date): # Generate expedition.yaml from the MPF file click.echo(f"Generating schedule from {mfp_file}...") mfp_to_yaml(mfp_file, start_date, expedition) - # TODO: this print needs to be updated # TODO: how to handle the ports?! Should be conditional on this kind of 'waypoint' being present in the MFP file. + # TODO: the schedule object should be able to take a special 'port' waypoint type, which is the same as a regular waypoint (to ensure compatibility) but without 'instruments' + # TODO: need to check this interacts as expected with the 'problems' module + # TODO: and new components need to be added to the 'plan' module to allow users to add ports and instruments to the schedule (keep in waypoints section but without instruments) + # TODO: but add and remove waypoint buttons should ignore ports click.echo( - "\n⚠️ The generated schedule does not contain TIME values or INSTRUMENT selections. ⚠️" - "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the schedule configuration, " - "\nOR edit 'expedition.yaml' and manually add the necessary time values and instrument selections under the 'schedule' heading." + "\n⚠️ The generated schedule does not contain INSTRUMENT selections. ⚠️" + "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the configuration, " + "\nOR edit 'expedition.yaml' and manually add the instrument selections under the 'schedule' heading." "\n\nIf editing 'expedition.yaml' manually:" - "\n\n🕒 Expected time format: 'YYYY-MM-DD HH:MM:SS' (e.g., '2023-10-20 01:00:00')." "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" ) From ff2f8f8a197edb75fa71d93153d49e6b7a4059f1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:05:15 +0200 Subject: [PATCH 04/12] add waypoint numbers (comments) to expedition.yaml --- src/virtualship/models/expedition.py | 19 +++++++- src/virtualship/static/expedition.yaml | 67 ++++++++++++++------------ 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index b72693731..07e4d184f 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -37,9 +37,24 @@ class Expedition(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") def to_yaml(self, file_path: str) -> None: - """Write exepedition object to yaml file.""" + """Write expedition object to yaml file, with waypoint number comments.""" + raw = yaml.dump(self.model_dump(by_alias=True), default_flow_style=False) + + breakpoint() + lines = raw.splitlines(keepends=True) + annotated = [] + waypoint_number = 0 + for line in lines: + if line.lstrip().startswith( + "- instrument:" + ): # TODO: unit test that this is how each waypoint is identified in the yaml dump + waypoint_number += 1 + indent = " " * (len(line) - len(line.lstrip())) + annotated.append(f"{indent}# Waypoint {waypoint_number}\n") + annotated.append(line) + with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) + file.writelines(annotated) @classmethod def from_yaml(cls, file_path: str) -> Expedition: diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index acb16dcf0..0f2c0c0b0 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -1,36 +1,5 @@ # see https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/working_with_expedition_yaml.html for more details on how to edit this file # -schedule: - waypoints: - - instrument: - - CTD - location: - latitude: 0 - longitude: 0 - time: 1998-01-01 00:00:00 - - instrument: - - DRIFTER - - CTD - location: - latitude: 0.01 - longitude: 0.01 - time: 1998-01-02 01:00:00 - - instrument: - - ARGO_FLOAT - location: - latitude: 0.02 - longitude: 0.02 - time: 1998-01-03 02:00:00 - - instrument: - - XBT - location: - latitude: 0.03 - longitude: 0.03 - time: 1998-01-04 03:00:00 - - location: - latitude: 0.03 - longitude: 0.03 - time: 1998-01-05 03:00:00 instruments_config: adcp_config: num_bins: 40 @@ -82,5 +51,41 @@ instruments_config: sensors: - TEMPERATURE - SALINITY +schedule: + waypoints: + # Waypoint 1 + - instrument: + - CTD + location: + latitude: 0 + longitude: 0 + time: 1998-01-01 00:00:00 + # Waypoint 2 + - instrument: + - DRIFTER + - CTD + location: + latitude: 0.01 + longitude: 0.01 + time: 1998-01-02 01:00:00 + # Waypoint 3 + - instrument: + - ARGO_FLOAT + location: + latitude: 0.02 + longitude: 0.02 + time: 1998-01-03 02:00:00 + # Waypoint 4 + - instrument: + - XBT + location: + latitude: 0.03 + longitude: 0.03 + time: 1998-01-04 03:00:00 + # Waypoint 5 + - location: + latitude: 0.03 + longitude: 0.03 + time: 1998-01-05 03:00:00 ship_config: ship_speed_knots: 10.0 From bf3a44249a4c45689d38b6fac6b6b1daf9a8b51b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:13:08 +0200 Subject: [PATCH 05/12] add unit test: waypoint field in yaml always starts with "- instrument" --- src/virtualship/models/expedition.py | 4 +-- tests/expedition/test_expedition.py | 38 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 07e4d184f..176220a5d 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -45,9 +45,7 @@ def to_yaml(self, file_path: str) -> None: annotated = [] waypoint_number = 0 for line in lines: - if line.lstrip().startswith( - "- instrument:" - ): # TODO: unit test that this is how each waypoint is identified in the yaml dump + if line.lstrip().startswith("- instrument:"): waypoint_number += 1 indent = " " * (len(line) - len(line.lstrip())) annotated.append(f"{indent}# Waypoint {waypoint_number}\n") diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 4bde12bdd..406ffbe4c 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -7,6 +7,7 @@ import pyproj import pytest import xarray as xr +import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( @@ -371,3 +372,40 @@ def test_all_instrument_configs_use_mixin(expedition): assert iconfig.__class__._instrument_type == iconfig._instrument_type, ( f"{iconfig.__class__.__name__}._instrument_type does not match its registered InstrumentType" ) + + +def test_waypoint_yaml_line() -> None: + """Each waypoint entry in the raw YAML dump should start with '- instrument:'.""" + base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=base_time, instrument=None), + Waypoint( + location=Location(1, 1), + time=base_time + timedelta(hours=1), + instrument=None, + ), + Waypoint( + location=Location(2, 2), + time=base_time + timedelta(hours=2), + instrument=["CTD"], + ), + ] + ) + raw = yaml.dump( + { + "schedule": { + "waypoints": [wp.model_dump(by_alias=True) for wp in schedule.waypoints] + } + }, + default_flow_style=False, + ) + + lines = [ + line for line in raw.splitlines() if line.lstrip().startswith("- instrument:") + ] + assert len(lines) == len(schedule.waypoints), ( + f"Expected {len(schedule.waypoints)} lines starting with '- instrument:' in the YAML dump, " + f"got {len(lines)}. The Waypoint field order or teminology may have changed. " + "Note this can have implications for the placement of waypoint number comments in Expedition.to_yaml()." + ) From a1d0b5e94bf6a76cd2ed2ce633f77935166aaf94 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:45:02 +0200 Subject: [PATCH 06/12] new Port class --- .../expedition/simulate_schedule.py | 7 ++- src/virtualship/models/__init__.py | 2 + src/virtualship/models/expedition.py | 63 ++++++++++++++----- src/virtualship/models/location.py | 29 +++++---- 4 files changed, 72 insertions(+), 29 deletions(-) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 6af9d80ce..93dd74416 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -16,6 +16,7 @@ from virtualship.models import ( Expedition, Location, + Port, Spacetime, Waypoint, ) @@ -247,7 +248,11 @@ def _get_underway_stationary_times( for i in range(1, int(npts) + 1) ] - def _make_measurements(self, waypoint: Waypoint) -> timedelta: + def _make_measurements(self, waypoint: Waypoint | Port) -> timedelta: + # port stops have no instruments + if isinstance(waypoint, Port): + return timedelta() + # if there are no instruments, there is no time cost if waypoint.instrument is None: return timedelta() diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index dd4b2bf14..b95544c89 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -8,6 +8,7 @@ DrifterConfig, Expedition, InstrumentsConfig, + Port, Schedule, SensorConfig, ShipConfig, @@ -23,6 +24,7 @@ __all__ = [ # noqa: RUF022 "Location", + "Port", "Schedule", "SensorConfig", "ShipConfig", diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 176220a5d..5b5193166 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -37,19 +37,8 @@ class Expedition(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") def to_yaml(self, file_path: str) -> None: - """Write expedition object to yaml file, with waypoint number comments.""" - raw = yaml.dump(self.model_dump(by_alias=True), default_flow_style=False) - - breakpoint() - lines = raw.splitlines(keepends=True) - annotated = [] - waypoint_number = 0 - for line in lines: - if line.lstrip().startswith("- instrument:"): - waypoint_number += 1 - indent = " " * (len(line) - len(line.lstrip())) - annotated.append(f"{indent}# Waypoint {waypoint_number}\n") - annotated.append(line) + """Write expedition object to yaml file, with port/waypoint number comments.""" + annotated = self._annotate() with open(file_path, "w") as file: file.writelines(annotated) @@ -66,6 +55,8 @@ def get_instruments(self) -> set[InstrumentType]: instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: + if isinstance(waypoint, Port): + continue if waypoint.instrument: for instrument in waypoint.instrument: if instrument: @@ -83,6 +74,36 @@ def get_instruments(self) -> set[InstrumentType]: "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." ) from e + def _annotate(self): + """Add port/waypoint comments/annotations to the expedition.yaml file.""" + assert isinstance(self.schedule.waypoints[0], Port) & isinstance( + self.schedule.waypoints[-1], Port + ), ( + "First and last waypoints must be Ports." + ) # commenting logic below assumes first and last waypoints are ports + + raw = yaml.dump(self.model_dump(by_alias=True), default_flow_style=False) + + lines = raw.splitlines(keepends=True) + annotated = [] + waypoint_number = 0 + for line in lines: + stripped = line.lstrip() + indent = " " * (len(line) - len(stripped)) + + # waypoints start with "- instrument:" and Ports start with "- location:" (no instrument field). + if stripped.startswith("- instrument:"): + waypoint_number += 1 + annotated.append(f"{indent}# Waypoint {waypoint_number}\n") + + if stripped.startswith("- location:"): + arrival_departure = "Departure" if waypoint_number == 0 else "Arrival" + annotated.append(f"{indent}# Port of {arrival_departure}\n") + + annotated.append(line) + + return annotated + class ShipConfig(pydantic.BaseModel): """Configuration of the ship.""" @@ -97,7 +118,7 @@ class ShipConfig(pydantic.BaseModel): class Schedule(pydantic.BaseModel): """Schedule of the virtual ship.""" - waypoints: list[Waypoint] + waypoints: list[Port | Waypoint] model_config = pydantic.ConfigDict(extra="forbid") @@ -150,6 +171,8 @@ def verify( ) from e for wp_i, wp in enumerate(self.waypoints): + if isinstance(wp, Port): + continue # ports are in harbour; skip bathymetry land check try: value = bathymetry_field.eval( np.float64(0.0), # time @@ -175,7 +198,8 @@ def verify( zip(self.waypoints, self.waypoints[1:], strict=False) ): stationkeeping_time = _calc_wp_stationkeeping_time( - wp.instrument, instruments_config + wp.instrument if isinstance(wp, Waypoint) else None, + instruments_config, ) time_to_reach = _calc_sail_time( @@ -201,6 +225,15 @@ def verify( print("... All good to go!") +class Port(pydantic.BaseModel): + """A port stop: a location the ship visits with no instrument deployments made.""" + + location: Location | None = None + time: datetime | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + class Waypoint(pydantic.BaseModel): """A Waypoint to sail to with an optional time and an optional instrument.""" diff --git a/src/virtualship/models/location.py b/src/virtualship/models/location.py index 793e5312c..1c40bb8b4 100644 --- a/src/virtualship/models/location.py +++ b/src/virtualship/models/location.py @@ -7,26 +7,29 @@ class Location: """A location on a sphere.""" - latitude: float - longitude: float + latitude: float | None = None + longitude: float | None = None def __post_init__(self) -> None: """ - Verify this location has valid latitude and longitude. + Verify this location has valid latitude and longitude if provided. :raises ValueError: If latitude and/or longitude are not valid. """ - if self.lat < -90: - raise ValueError("Latitude cannot be smaller than -90.") - if self.lat > 90: - raise ValueError("Latitude cannot be larger than 90.") - if self.lon < -180: - raise ValueError("Longitude cannot be smaller than -180.") - if self.lon > 360: - raise ValueError("Longitude cannot be larger than 360.") + if self.lat is not None: + if self.lat < -90: + raise ValueError("Latitude cannot be smaller than -90.") + if self.lat > 90: + raise ValueError("Latitude cannot be larger than 90.") + + if self.lon is not None: + if self.lon < -180: + raise ValueError("Longitude cannot be smaller than -180.") + if self.lon > 360: + raise ValueError("Longitude cannot be larger than 360.") @property - def lat(self) -> float: + def lat(self) -> float | None: """ Shorthand for latitude variable. @@ -35,7 +38,7 @@ def lat(self) -> float: return self.latitude @property - def lon(self) -> float: + def lon(self) -> float | None: """ Shorthand for longitude variable. From f6bca0faca6a3ff67fb12884759a91781d5255b8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:17:08 +0200 Subject: [PATCH 07/12] ingest and validate mfp exports with/without (or mixture of) ports --- src/virtualship/utils.py | 116 +++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 9126c72d6..77e5679ee 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -208,8 +208,18 @@ def _load_mfpexport(file_path): ) from e +def _create_port_row(columns, port_type): + """Generate a single placeholder row for missing departure/arrival ports.""" + row = {col: None for col in columns} + row["Station"] = port_type + row["Type"] = port_type + return pd.DataFrame([row]) + + def _validate_mfpdata(file_path): """Load and validate MFP CruiseData export.""" + errmsg_supplement = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." + mfp_data = _load_mfpexport(file_path) # clean up column names @@ -237,33 +247,57 @@ def _validate_mfpdata(file_path): if missing_columns: raise ValueError( f"Error: Found columns {list(actual_columns)}, but expected columns {list(expected_columns)}. " - "Are you sure that you're using the correct export from MFP?" + "Are you sure that you're using the correct export from MFP?\n\n" + + errmsg_supplement ) extra_columns = actual_columns - expected_columns if extra_columns: + # TODO: as mentioned below, propagate this warning to the user via the click.echo() output in the `virtualship init` command? warnings.warn( f"Found additional unexpected columns {list(extra_columns)}. " - "Manually added columns have no effect. " - "If the MFP export format changed, please submit an issue: " - "https://github.com/OceanParcels/virtualship/issues.", + "Manually added columns have no effect. " + errmsg_supplement, stacklevel=2, ) - # Drop unexpected columns - mfp_data = mfp_data[list(expected_columns)] + # Convert latitude and longitude to floats, handling commas and missing values safely + for coord in ["Latitude", "Longitude"]: + if mfp_data[coord].dtype in ["object", "string"]: + mfp_data[coord] = pd.to_numeric( + mfp_data[coord].astype(str).str.replace(",", "."), errors="coerce" + ) - # Convert latitude and longitude to floats, replacing commas with dots - # Handles case when the latitude and longitude have decimals with commas - if mfp_data["Latitude"].dtype in ["object", "string"]: - mfp_data["Latitude"] = mfp_data["Latitude"].apply( - lambda x: float(x.replace(",", ".")) + # check for missing departure/arrival ports and add placeholders if necessary + # check against both 'Station' and 'Type' columns; variations can occur when importing to MFP before re-exporting + has_departure = ( + "Departure Port" in mfp_data["Station"].values + or "Departure Port" in mfp_data["Type"].values + ) + has_arrival = ( + "Arrival Port" in mfp_data["Station"].values + or "Arrival Port" in mfp_data["Type"].values + ) + if not has_departure or not has_arrival: + # TODO: propagate the warning to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. + warnings.warn( + "The MFP export is missing either a 'Departure Port' or 'Arrival Port', or both. " + "Any missing port will be replaced with a placeholder in `expedition.yaml` but will be ignored in the simulation. " + "The prescribed date will be used for Waypoint #1 instead. " + "If you believe this warning is wrong (i.e. you have selected departure/arrival ports), and " + + errmsg_supplement.replace("If ", ""), + stacklevel=2, ) - if mfp_data["Longitude"].dtype in ["object", "string"]: - mfp_data["Longitude"] = mfp_data["Longitude"].apply( - lambda x: float(x.replace(",", ".")) - ) + if not has_departure: + dept_row = _create_port_row(expected_columns, "Departure Port") + mfp_data = pd.concat([dept_row, mfp_data], ignore_index=True) # first row + + if not has_arrival: + arr_row = _create_port_row(expected_columns, "Arrival Port") + mfp_data = pd.concat([mfp_data, arr_row], ignore_index=True) # last row + + # Drop unexpected columns + mfp_data = mfp_data[list(expected_columns)] # convert 'Travel Time to Next' and 'Time at Station' to timedelta mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( @@ -274,9 +308,10 @@ def _validate_mfpdata(file_path): ) # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column - mfp_data["Total Time"] = ( - mfp_data["Travel Time to Next"] + mfp_data["Time at Station"] - ) + # add 0 when Time at Station is NaN, to avoid NaT in Total Time, but not to Travel Time to keep NaT at the arrival port + mfp_data["Total Time"] = mfp_data["Travel Time to Next"] + mfp_data[ + "Time at Station" + ].fillna(pd.Timedelta(0)) return mfp_data @@ -288,6 +323,7 @@ def mfp_to_yaml(file_path: str, start_date: str, output_path: str): Expedition, InstrumentsConfig, Location, + Port, Schedule, Waypoint, ) @@ -295,22 +331,42 @@ def mfp_to_yaml(file_path: str, start_date: str, output_path: str): # Read data from file mfp_data = _validate_mfpdata(file_path) - # Generate waypoints + # Generate ports/waypoints waypoints = [] current_time, previous_timedelta = start_date, None for i, row in mfp_data.iterrows(): if i > 0: current_time += previous_timedelta - waypoints.append( - Waypoint( - instrument=None, - location=Location(latitude=row["Latitude"], longitude=row["Longitude"]), - time=current_time, + is_port = "Port" in row["Station"] or "Port" in row["Type"] + + if is_port: + has_latlon = not pd.isna(row["Latitude"]) and not pd.isna( + row["Longitude"] + ) # indicates that the port has been set in MFP / is not a placeholder + + waypoints.append( + Port( + location=Location( + latitude=row["Latitude"], longitude=row["Longitude"] + ), + time=current_time if has_latlon else None, + ) ) + else: + waypoints.append( + Waypoint( + instrument=None, + location=Location( + latitude=row["Latitude"], longitude=row["Longitude"] + ), + time=current_time, + ) + ) + + # store total timedelta for next iteration + previous_timedelta = ( + row["Total Time"] if row["Total Time"] is not pd.NaT else timedelta(0) ) - previous_timedelta = row[ - "Total Time" - ] # store total timedelta for next iteration # Create Schedule object schedule = Schedule( @@ -338,8 +394,8 @@ def mfp_to_yaml(file_path: str, start_date: str, output_path: str): def _mfp_string_to_timedelta(value: str) -> timedelta: """Handle MFP export string format (e.g., "0d 13h 13m").""" - if pd.isna(value): # last waypoint has no travel time to next, so will be NaN - return timedelta(0) + if pd.isna(value): # last waypoint/missing ports have NaN/None travel time + return value # return None value = value.replace("d", ":").replace("h", ":").replace("m", "") days, hours, minutes = map(int, value.split(":")) @@ -653,7 +709,7 @@ def _calc_sail_time( def _calc_wp_stationkeeping_time( - wp_instrument_types: list, + wp_instrument_types: list | None, instruments_config: InstrumentsConfig, instrument_config_map: dict = INSTRUMENT_CONFIG_MAP, ) -> timedelta: From e91686b2afa501bbd00e6ce167ed0f7153df2c64 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:17:30 +0200 Subject: [PATCH 08/12] update static expedition.yaml with ports API --- src/virtualship/static/expedition.yaml | 41 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 0f2c0c0b0..ad04aa67f 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -53,39 +53,50 @@ instruments_config: - SALINITY schedule: waypoints: + # Port of Departure + - location: + latitude: 0 + longitude: 0 + time: 1998-01-01 00:00:00 # Waypoint 1 - instrument: - CTD location: - latitude: 0 - longitude: 0 - time: 1998-01-01 00:00:00 + latitude: 0.01 + longitude: 0.01 + time: 1998-01-02 00:00:00 # Waypoint 2 - instrument: - DRIFTER - CTD location: - latitude: 0.01 - longitude: 0.01 - time: 1998-01-02 01:00:00 + latitude: 0.02 + longitude: 0.02 + time: 1998-01-03 01:00:00 # Waypoint 3 - instrument: - ARGO_FLOAT location: - latitude: 0.02 - longitude: 0.02 - time: 1998-01-03 02:00:00 + latitude: 0.03 + longitude: 0.03 + time: 1998-01-04 02:00:00 # Waypoint 4 - instrument: - XBT location: - latitude: 0.03 - longitude: 0.03 - time: 1998-01-04 03:00:00 + latitude: 0.04 + longitude: 0.04 + time: 1998-01-05 03:00:00 # Waypoint 5 - - location: - latitude: 0.03 - longitude: 0.03 + - instrument: [] + location: + latitude: 0.05 + longitude: 0.05 time: 1998-01-05 03:00:00 + # Port of Arrival + - location: + latitude: 0.06 + longitude: 0.06 + time: 1998-01-06 03:00:00 ship_config: ship_speed_knots: 10.0 From 03da307f52d03766a7878f052025a9f616165c3f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:19:49 +0200 Subject: [PATCH 09/12] fix timings in static expedition.yaml --- src/virtualship/static/expedition.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index ad04aa67f..1e201543c 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -92,11 +92,11 @@ schedule: location: latitude: 0.05 longitude: 0.05 - time: 1998-01-05 03:00:00 + time: 1998-01-06 04:00:00 # Port of Arrival - location: latitude: 0.06 longitude: 0.06 - time: 1998-01-06 03:00:00 + time: 1998-01-07 05:00:00 ship_config: ship_speed_knots: 10.0 From 8cf91de1587e1bdae7c0abeb0e4d73441bb3d0f0 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:47:46 +0200 Subject: [PATCH 10/12] move `virtualship init` logic to new initialise.py module --- src/virtualship/cli/_initialise.py | 287 +++++++++++++++++++++++++++++ src/virtualship/cli/commands.py | 41 +---- src/virtualship/utils.py | 238 ------------------------ 3 files changed, 290 insertions(+), 276 deletions(-) create mode 100644 src/virtualship/cli/_initialise.py diff --git a/src/virtualship/cli/_initialise.py b/src/virtualship/cli/_initialise.py new file mode 100644 index 000000000..8a8db8ae6 --- /dev/null +++ b/src/virtualship/cli/_initialise.py @@ -0,0 +1,287 @@ +import os +import warnings +from datetime import timedelta +from functools import lru_cache +from importlib.resources import files +from pathlib import Path + +import click +import pandas as pd +import yaml + +from virtualship.utils import ( + EXPEDITION, +) + + +def _initialise( + path: str | Path, from_mfp: str | None = None, start_date: str | None = None +): + path = Path(path) + path.mkdir(exist_ok=True) + + expedition = path / EXPEDITION + + if expedition.exists(): + raise FileExistsError( + f"File '{expedition}' already exist. Please remove it or choose another directory." + ) + + if from_mfp: + mfp_file = Path(from_mfp) + # Generate expedition.yaml from the MPF file + click.echo(f"Generating schedule from {mfp_file}...") + _mfp_to_yaml(mfp_file, start_date, expedition) + # TODO: need to check this interacts as expected with the 'problems' module + # TODO: and new components need to be added to the 'plan' module to allow users to add ports and instruments to the schedule (keep in waypoints section but without instruments) + # TODO: but add and remove waypoint buttons should ignore ports + # TODO: update relevant docs + #! TODO: `virtualship init` methods are becoming long and complex. Consider refactoring into a separate module for clarity and maintainability (in `virtualship/cli/_init.py`). + # though, consider confusion of having both `_init.py` and `init.py` in the same directory. Maybe `_init.py` should be renamed to `_init_command.py` or similar. + # TODO: add a check to see if any instruments are added to a port waypoint (shouldn't be possible via MFP export but in case someone manually edits the expedition.yaml to add instruments to a port waypoint). If so, raise an error and ask user to remove them. + #! TODO: see utils.py: propagate the warnings to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. + + click.echo( + "\n⚠️ The generated schedule does not contain INSTRUMENT selections. ⚠️" + "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the configuration, " + "\nOR edit 'expedition.yaml' and manually add the instrument selections under the 'schedule' heading." + "\n\nIf editing 'expedition.yaml' manually:" + "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." + f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" + ) + else: + # Create a default example expedition YAML + expedition.write_text(_get_example_expedition()) + + click.echo(f"Created '{expedition.name}' at {path}.") + + +def _mfp_to_yaml(file_path: str, start_date: str, output_path: str): + """Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version.""" + # avoid circular imports + from virtualship.models import ( + Expedition, + InstrumentsConfig, + Location, + Port, + Schedule, + Waypoint, + ) + + # Read data from file + mfp_data = _validate_mfp_data(file_path) + + # Generate ports/waypoints + waypoints = [] + current_time, previous_timedelta = start_date, None + for i, row in mfp_data.iterrows(): + if i > 0: + current_time += previous_timedelta + is_port = "Port" in row["Station"] or "Port" in row["Type"] + + if is_port: + has_latlon = not pd.isna(row["Latitude"]) and not pd.isna( + row["Longitude"] + ) # indicates that the port has been set in MFP / is not a placeholder + + waypoints.append( + Port( + location=Location( + latitude=row["Latitude"], longitude=row["Longitude"] + ), + time=current_time if has_latlon else None, + ) + ) + else: + waypoints.append( + Waypoint( + instrument=None, + location=Location( + latitude=row["Latitude"], longitude=row["Longitude"] + ), + time=current_time, + ) + ) + + # store total timedelta for next iteration + previous_timedelta = ( + row["Total Time"] if row["Total Time"] is not pd.NaT else timedelta(0) + ) + + # Create Schedule object + schedule = Schedule( + waypoints=waypoints, + ) + + # extract instruments config from static + instruments_config = InstrumentsConfig.model_validate( + yaml.safe_load(_get_example_expedition()).get("instruments_config") + ) + + # extract ship config from static + ship_config = yaml.safe_load(_get_example_expedition()).get("ship_config") + # combine to Expedition object + expedition = Expedition( + schedule=schedule, + instruments_config=instruments_config, + ship_config=ship_config, + ) + + # Save to YAML file + expedition.to_yaml(output_path) + + +def _validate_mfp_data(file_path): + """Load and validate MFP CruiseData export.""" + errmsg_supplement = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." + + mfp_data = _load_mfp_export(file_path) + + # clean up column names + mfp_data.columns = mfp_data.columns.astype(str).str.strip() + mfp_data = mfp_data.loc[ + :, ~mfp_data.columns.str.startswith("Unnamed") & (mfp_data.columns != "") + ] + + expected_columns = { + "Station", + "Type", + "Latitude", + "Longitude", + "Sea Depth", + "Time at Station", + "Travel Time to Next", + "Distance to Next (NM)", + "Ship Speed (kn)", + "EEZ", + } + + actual_columns = set(mfp_data.columns) + + missing_columns = expected_columns - actual_columns + if missing_columns: + raise ValueError( + f"Error: Found columns {list(actual_columns)}, but expected columns {list(expected_columns)}. " + "Are you sure that you're using the correct export from MFP?\n\n" + + errmsg_supplement + ) + + extra_columns = actual_columns - expected_columns + if extra_columns: + # TODO: as mentioned below, propagate this warning to the user via the click.echo() output in the `virtualship init` command? + warnings.warn( + f"Found additional unexpected columns {list(extra_columns)}. " + "Manually added columns have no effect. " + errmsg_supplement, + stacklevel=2, + ) + + # Convert latitude and longitude to floats, handling commas and missing values safely + for coord in ["Latitude", "Longitude"]: + if mfp_data[coord].dtype in ["object", "string"]: + mfp_data[coord] = pd.to_numeric( + mfp_data[coord].astype(str).str.replace(",", "."), errors="coerce" + ) + + # check for missing departure/arrival ports and add placeholders if necessary + # check against both 'Station' and 'Type' columns; variations can occur when importing to MFP before re-exporting + has_departure = ( + "Departure Port" in mfp_data["Station"].values + or "Departure Port" in mfp_data["Type"].values + ) + has_arrival = ( + "Arrival Port" in mfp_data["Station"].values + or "Arrival Port" in mfp_data["Type"].values + ) + if not has_departure or not has_arrival: + # TODO: propagate the warning to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. + warnings.warn( + "The MFP export is missing either a 'Departure Port' or 'Arrival Port', or both. " + "Any missing port will be replaced with a placeholder in `expedition.yaml` but will be ignored in the simulation. " + "The prescribed date will be used for Waypoint #1 instead. " + "If you believe this warning is wrong (i.e. you have selected departure/arrival ports), and " + + errmsg_supplement.replace("If ", ""), + stacklevel=2, + ) + + if not has_departure: + dept_row = _create_port_row(expected_columns, "Departure Port") + mfp_data = pd.concat([dept_row, mfp_data], ignore_index=True) # first row + + if not has_arrival: + arr_row = _create_port_row(expected_columns, "Arrival Port") + mfp_data = pd.concat([mfp_data, arr_row], ignore_index=True) # last row + + # Drop unexpected columns + mfp_data = mfp_data[list(expected_columns)] + + # convert 'Travel Time to Next' and 'Time at Station' to timedelta + mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( + lambda x: _mfp_string_to_timedelta(x) + ) + mfp_data["Time at Station"] = mfp_data["Time at Station"].apply( + lambda x: _mfp_string_to_timedelta(x) + ) + + # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column + # add 0 when Time at Station is NaN, to avoid NaT in Total Time, but not to Travel Time to keep NaT at the arrival port + mfp_data["Total Time"] = mfp_data["Travel Time to Next"] + mfp_data[ + "Time at Station" + ].fillna(pd.Timedelta(0)) + + return mfp_data + + +def _load_mfp_export(file_path): + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + try: + df = pd.read_excel(file_path) + return df.dropna(how="all", axis=1) # drop empty columns + + except Exception as e: + raise RuntimeError( + "Could not read coordinates data from the provided file. " + "Ensure it is an exported .xlsx file from MFP." + ) from e + + +def _create_port_row(columns, port_type): + """Generate a single placeholder row for missing departure/arrival ports.""" + row = {col: None for col in columns} + row["Station"] = port_type + row["Type"] = port_type + return pd.DataFrame([row]) + + +def _mfp_string_to_timedelta(value: str) -> timedelta: + """Handle MFP export string format (e.g., "0d 13h 13m").""" + if pd.isna(value): # last waypoint/missing ports have NaN/None travel time + return value # return None + + value = value.replace("d", ":").replace("h", ":").replace("m", "") + days, hours, minutes = map(int, value.split(":")) + return timedelta(days=days, hours=hours, minutes=minutes) + + +def _load_static_file(name: str) -> str: + """Load static file from the ``virtualship.static`` module by file name.""" + return files("virtualship.static").joinpath(name).read_text(encoding="utf-8") + + +@lru_cache(None) +@lru_cache(None) +def _get_example_expedition() -> str: + """Get the example unified expedition configuration file.""" + return _load_static_file(EXPEDITION) + + +def _validate_start_date(ctx, param, value): + """Callback to enforce and validate --start-date when --from-mfp is used.""" + if ctx.params.get("from_mfp"): + if not value: + raise click.BadParameter( + "The '--start-date' option is required when using '--from-mfp'." + "\n\nExpected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00." + ) + return value diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 2ecf71255..d958df169 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -2,15 +2,12 @@ import click +from virtualship.cli._initialise import _initialise, _validate_start_date from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, - EXPEDITION, - get_example_expedition, - mfp_to_yaml, - validate_start_date, ) @@ -31,7 +28,7 @@ "--start-date", type=click.DateTime(formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"]), default=None, - callback=validate_start_date, + callback=_validate_start_date, help="The departure/start date of the expedition (required when using --from-mfp). " "Expected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00.", ) @@ -41,39 +38,7 @@ def init(path, from_mfp, start_date): If --mfp-file is provided, it will generate the expedition.yaml from the MPF file instead. """ - path = Path(path) - path.mkdir(exist_ok=True) - - expedition = path / EXPEDITION - - if expedition.exists(): - raise FileExistsError( - f"File '{expedition}' already exist. Please remove it or choose another directory." - ) - - if from_mfp: - mfp_file = Path(from_mfp) - # Generate expedition.yaml from the MPF file - click.echo(f"Generating schedule from {mfp_file}...") - mfp_to_yaml(mfp_file, start_date, expedition) - # TODO: how to handle the ports?! Should be conditional on this kind of 'waypoint' being present in the MFP file. - # TODO: the schedule object should be able to take a special 'port' waypoint type, which is the same as a regular waypoint (to ensure compatibility) but without 'instruments' - # TODO: need to check this interacts as expected with the 'problems' module - # TODO: and new components need to be added to the 'plan' module to allow users to add ports and instruments to the schedule (keep in waypoints section but without instruments) - # TODO: but add and remove waypoint buttons should ignore ports - click.echo( - "\n⚠️ The generated schedule does not contain INSTRUMENT selections. ⚠️" - "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the configuration, " - "\nOR edit 'expedition.yaml' and manually add the instrument selections under the 'schedule' heading." - "\n\nIf editing 'expedition.yaml' manually:" - "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." - f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" - ) - else: - # Create a default example expedition YAML - expedition.write_text(get_example_expedition()) - - click.echo(f"Created '{expedition.name}' at {path}.") + _initialise(Path(path), from_mfp, start_date) @click.command() diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 77e5679ee..9441defd5 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -2,16 +2,11 @@ import glob import hashlib -import os import re -import warnings from datetime import datetime, timedelta -from functools import lru_cache -from importlib.resources import files from pathlib import Path from typing import TYPE_CHECKING, Literal, TextIO -import click import copernicusmarine import numpy as np import parcels @@ -29,7 +24,6 @@ from virtualship.models.checkpoint import Checkpoint from virtualship.models.expedition import SensorConfig -import pandas as pd import yaml from pydantic import BaseModel from yaspin import Spinner @@ -158,18 +152,6 @@ def decorator(cls): # ===================================================== -def load_static_file(name: str) -> str: - """Load static file from the ``virtualship.static`` module by file name.""" - return files("virtualship.static").joinpath(name).read_text(encoding="utf-8") - - -@lru_cache(None) -@lru_cache(None) -def get_example_expedition() -> str: - """Get the example unified expedition configuration file.""" - return load_static_file(EXPEDITION) - - def _dump_yaml(model: BaseModel, stream: TextIO) -> str | None: """Dump a pydantic model to a yaml string.""" return yaml.safe_dump( @@ -182,226 +164,6 @@ def _generic_load_yaml(data: str, model: BaseModel) -> BaseModel: return model.model_validate(yaml.safe_load(data)) -def validate_start_date(ctx, param, value): - """Callback to enforce and validate --start-date when --from-mfp is used.""" - if ctx.params.get("from_mfp"): - if not value: - raise click.BadParameter( - "The '--start-date' option is required when using '--from-mfp'." - "\n\nExpected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00." - ) - return value - - -def _load_mfpexport(file_path): - if not os.path.isfile(file_path): - raise FileNotFoundError(f"File not found: {file_path}") - - try: - df = pd.read_excel(file_path) - return df.dropna(how="all", axis=1) # drop empty columns - - except Exception as e: - raise RuntimeError( - "Could not read coordinates data from the provided file. " - "Ensure it is an exported .xlsx file from MFP." - ) from e - - -def _create_port_row(columns, port_type): - """Generate a single placeholder row for missing departure/arrival ports.""" - row = {col: None for col in columns} - row["Station"] = port_type - row["Type"] = port_type - return pd.DataFrame([row]) - - -def _validate_mfpdata(file_path): - """Load and validate MFP CruiseData export.""" - errmsg_supplement = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." - - mfp_data = _load_mfpexport(file_path) - - # clean up column names - mfp_data.columns = mfp_data.columns.astype(str).str.strip() - mfp_data = mfp_data.loc[ - :, ~mfp_data.columns.str.startswith("Unnamed") & (mfp_data.columns != "") - ] - - expected_columns = { - "Station", - "Type", - "Latitude", - "Longitude", - "Sea Depth", - "Time at Station", - "Travel Time to Next", - "Distance to Next (NM)", - "Ship Speed (kn)", - "EEZ", - } - - actual_columns = set(mfp_data.columns) - - missing_columns = expected_columns - actual_columns - if missing_columns: - raise ValueError( - f"Error: Found columns {list(actual_columns)}, but expected columns {list(expected_columns)}. " - "Are you sure that you're using the correct export from MFP?\n\n" - + errmsg_supplement - ) - - extra_columns = actual_columns - expected_columns - if extra_columns: - # TODO: as mentioned below, propagate this warning to the user via the click.echo() output in the `virtualship init` command? - warnings.warn( - f"Found additional unexpected columns {list(extra_columns)}. " - "Manually added columns have no effect. " + errmsg_supplement, - stacklevel=2, - ) - - # Convert latitude and longitude to floats, handling commas and missing values safely - for coord in ["Latitude", "Longitude"]: - if mfp_data[coord].dtype in ["object", "string"]: - mfp_data[coord] = pd.to_numeric( - mfp_data[coord].astype(str).str.replace(",", "."), errors="coerce" - ) - - # check for missing departure/arrival ports and add placeholders if necessary - # check against both 'Station' and 'Type' columns; variations can occur when importing to MFP before re-exporting - has_departure = ( - "Departure Port" in mfp_data["Station"].values - or "Departure Port" in mfp_data["Type"].values - ) - has_arrival = ( - "Arrival Port" in mfp_data["Station"].values - or "Arrival Port" in mfp_data["Type"].values - ) - if not has_departure or not has_arrival: - # TODO: propagate the warning to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. - warnings.warn( - "The MFP export is missing either a 'Departure Port' or 'Arrival Port', or both. " - "Any missing port will be replaced with a placeholder in `expedition.yaml` but will be ignored in the simulation. " - "The prescribed date will be used for Waypoint #1 instead. " - "If you believe this warning is wrong (i.e. you have selected departure/arrival ports), and " - + errmsg_supplement.replace("If ", ""), - stacklevel=2, - ) - - if not has_departure: - dept_row = _create_port_row(expected_columns, "Departure Port") - mfp_data = pd.concat([dept_row, mfp_data], ignore_index=True) # first row - - if not has_arrival: - arr_row = _create_port_row(expected_columns, "Arrival Port") - mfp_data = pd.concat([mfp_data, arr_row], ignore_index=True) # last row - - # Drop unexpected columns - mfp_data = mfp_data[list(expected_columns)] - - # convert 'Travel Time to Next' and 'Time at Station' to timedelta - mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( - lambda x: _mfp_string_to_timedelta(x) - ) - mfp_data["Time at Station"] = mfp_data["Time at Station"].apply( - lambda x: _mfp_string_to_timedelta(x) - ) - - # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column - # add 0 when Time at Station is NaN, to avoid NaT in Total Time, but not to Travel Time to keep NaT at the arrival port - mfp_data["Total Time"] = mfp_data["Travel Time to Next"] + mfp_data[ - "Time at Station" - ].fillna(pd.Timedelta(0)) - - return mfp_data - - -def mfp_to_yaml(file_path: str, start_date: str, output_path: str): - """Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version.""" - # avoid circular imports - from virtualship.models import ( - Expedition, - InstrumentsConfig, - Location, - Port, - Schedule, - Waypoint, - ) - - # Read data from file - mfp_data = _validate_mfpdata(file_path) - - # Generate ports/waypoints - waypoints = [] - current_time, previous_timedelta = start_date, None - for i, row in mfp_data.iterrows(): - if i > 0: - current_time += previous_timedelta - is_port = "Port" in row["Station"] or "Port" in row["Type"] - - if is_port: - has_latlon = not pd.isna(row["Latitude"]) and not pd.isna( - row["Longitude"] - ) # indicates that the port has been set in MFP / is not a placeholder - - waypoints.append( - Port( - location=Location( - latitude=row["Latitude"], longitude=row["Longitude"] - ), - time=current_time if has_latlon else None, - ) - ) - else: - waypoints.append( - Waypoint( - instrument=None, - location=Location( - latitude=row["Latitude"], longitude=row["Longitude"] - ), - time=current_time, - ) - ) - - # store total timedelta for next iteration - previous_timedelta = ( - row["Total Time"] if row["Total Time"] is not pd.NaT else timedelta(0) - ) - - # Create Schedule object - schedule = Schedule( - waypoints=waypoints, - ) - - # extract instruments config from static - instruments_config = InstrumentsConfig.model_validate( - yaml.safe_load(get_example_expedition()).get("instruments_config") - ) - - # extract ship config from static - ship_config = yaml.safe_load(get_example_expedition()).get("ship_config") - - # combine to Expedition object - expedition = Expedition( - schedule=schedule, - instruments_config=instruments_config, - ship_config=ship_config, - ) - - # Save to YAML file - expedition.to_yaml(output_path) - - -def _mfp_string_to_timedelta(value: str) -> timedelta: - """Handle MFP export string format (e.g., "0d 13h 13m").""" - if pd.isna(value): # last waypoint/missing ports have NaN/None travel time - return value # return None - - value = value.replace("d", ":").replace("h", ":").replace("m", "") - days, hours, minutes = map(int, value.split(":")) - return timedelta(days=days, hours=hours, minutes=minutes) - - def _validate_numeric_to_timedelta( value: int | float | timedelta, unit: Literal["minutes", "days"] ) -> timedelta: From 46e78283cfcf3fd615d482b1e09238acd80a8f62 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:48:39 +0200 Subject: [PATCH 11/12] update init docstring --- src/virtualship/cli/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index d958df169..3442fb1bf 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -36,7 +36,7 @@ def init(path, from_mfp, start_date): """ Initialize a directory for a new expedition, with an expedition.yaml file. - If --mfp-file is provided, it will generate the expedition.yaml from the MPF file instead. + If --mfp-file is provided (and --start-date is also provided), it will generate the expedition.yaml from the MPF file instead. """ _initialise(Path(path), from_mfp, start_date) From 02a1dd7edd2e875f875917edeabdc0118352b900 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:17:41 +0200 Subject: [PATCH 12/12] refactor _initialise.py --- src/virtualship/cli/_initialise.py | 201 +++++++++++++---------------- 1 file changed, 90 insertions(+), 111 deletions(-) diff --git a/src/virtualship/cli/_initialise.py b/src/virtualship/cli/_initialise.py index 8a8db8ae6..41145c8fb 100644 --- a/src/virtualship/cli/_initialise.py +++ b/src/virtualship/cli/_initialise.py @@ -1,4 +1,5 @@ import os +import re import warnings from datetime import timedelta from functools import lru_cache @@ -9,9 +10,17 @@ import pandas as pd import yaml -from virtualship.utils import ( - EXPEDITION, +from virtualship.models import ( + Expedition, + InstrumentsConfig, + Location, + Port, + Schedule, + Waypoint, ) +from virtualship.utils import EXPEDITION + +ERR_SUPPLEMENT = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." def _initialise( @@ -24,126 +33,98 @@ def _initialise( if expedition.exists(): raise FileExistsError( - f"File '{expedition}' already exist. Please remove it or choose another directory." + f"File '{expedition}' already exists. Please remove it or choose another directory." ) if from_mfp: mfp_file = Path(from_mfp) - # Generate expedition.yaml from the MPF file click.echo(f"Generating schedule from {mfp_file}...") - _mfp_to_yaml(mfp_file, start_date, expedition) - # TODO: need to check this interacts as expected with the 'problems' module - # TODO: and new components need to be added to the 'plan' module to allow users to add ports and instruments to the schedule (keep in waypoints section but without instruments) - # TODO: but add and remove waypoint buttons should ignore ports - # TODO: update relevant docs - #! TODO: `virtualship init` methods are becoming long and complex. Consider refactoring into a separate module for clarity and maintainability (in `virtualship/cli/_init.py`). - # though, consider confusion of having both `_init.py` and `init.py` in the same directory. Maybe `_init.py` should be renamed to `_init_command.py` or similar. - # TODO: add a check to see if any instruments are added to a port waypoint (shouldn't be possible via MFP export but in case someone manually edits the expedition.yaml to add instruments to a port waypoint). If so, raise an error and ask user to remove them. - #! TODO: see utils.py: propagate the warnings to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. + # catch warnings raised to propagate them via click.echo + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + _mfp_to_yaml(mfp_file, start_date, expedition) + + indent = " " * 4 click.echo( "\n⚠️ The generated schedule does not contain INSTRUMENT selections. ⚠️" "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the configuration, " "\nOR edit 'expedition.yaml' and manually add the instrument selections under the 'schedule' heading." "\n\nIf editing 'expedition.yaml' manually:" - "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." - f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" + "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." + f"\n\n{indent * 4}waypoints:\n{indent * 4}- instrument:\n{indent * 5}- CTD\n{indent * 5}- ARGO_FLOAT\n" ) + + # output captured warnings to the terminal + if captured_warnings: + click.echo("\n❗️ WARNINGS:") + for w in captured_warnings: + click.echo(f"{indent}• {w.message}") + click.echo( + f"\n{indent}If you believe any of these warnings are incorrect (e.g. you have selected departure/arrival ports), and {ERR_SUPPLEMENT.replace('If ', '')}\n" + ) else: - # Create a default example expedition YAML expedition.write_text(_get_example_expedition()) click.echo(f"Created '{expedition.name}' at {path}.") -def _mfp_to_yaml(file_path: str, start_date: str, output_path: str): - """Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version.""" - # avoid circular imports - from virtualship.models import ( - Expedition, - InstrumentsConfig, - Location, - Port, - Schedule, - Waypoint, - ) - - # Read data from file +def _mfp_to_yaml(file_path: Path, start_date: str, output_path: Path): + """Generates an expedition.yaml file from MFP Excel export.""" mfp_data = _validate_mfp_data(file_path) - # Generate ports/waypoints + # convert start_date string to datetime object if needed + if isinstance(start_date, str): + current_time = pd.to_datetime(start_date) + else: + current_time = start_date + waypoints = [] - current_time, previous_timedelta = start_date, None + previous_timedelta = None + for i, row in mfp_data.iterrows(): if i > 0: current_time += previous_timedelta - is_port = "Port" in row["Station"] or "Port" in row["Type"] - if is_port: - has_latlon = not pd.isna(row["Latitude"]) and not pd.isna( - row["Longitude"] - ) # indicates that the port has been set in MFP / is not a placeholder + is_port = "Port" in str(row["Station"]) or "Port" in str(row["Type"]) + lat = None if pd.isna(row["Latitude"]) else float(row["Latitude"]) + lon = None if pd.isna(row["Longitude"]) else float(row["Longitude"]) + loc = Location(latitude=lat, longitude=lon) + if is_port: + has_latlon = lat is not None and lon is not None waypoints.append( - Port( - location=Location( - latitude=row["Latitude"], longitude=row["Longitude"] - ), - time=current_time if has_latlon else None, - ) + Port(location=loc, time=current_time if has_latlon else None) ) else: - waypoints.append( - Waypoint( - instrument=None, - location=Location( - latitude=row["Latitude"], longitude=row["Longitude"] - ), - time=current_time, - ) - ) + waypoints.append(Waypoint(instrument=None, location=loc, time=current_time)) - # store total timedelta for next iteration previous_timedelta = ( - row["Total Time"] if row["Total Time"] is not pd.NaT else timedelta(0) + row["Total Time"] if pd.notna(row["Total Time"]) else timedelta(0) ) - # Create Schedule object - schedule = Schedule( - waypoints=waypoints, - ) - - # extract instruments config from static - instruments_config = InstrumentsConfig.model_validate( - yaml.safe_load(_get_example_expedition()).get("instruments_config") - ) - - # extract ship config from static - ship_config = yaml.safe_load(_get_example_expedition()).get("ship_config") - # combine to Expedition object + # build and dump expedition YAML + static_yaml = yaml.safe_load(_get_example_expedition()) expedition = Expedition( - schedule=schedule, - instruments_config=instruments_config, - ship_config=ship_config, + schedule=Schedule(waypoints=waypoints), + instruments_config=InstrumentsConfig.model_validate( + static_yaml.get("instruments_config") + ), + ship_config=static_yaml.get("ship_config"), ) - - # Save to YAML file expedition.to_yaml(output_path) -def _validate_mfp_data(file_path): +def _validate_mfp_data(file_path: Path) -> pd.DataFrame: """Load and validate MFP CruiseData export.""" - errmsg_supplement = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." - mfp_data = _load_mfp_export(file_path) # clean up column names mfp_data.columns = mfp_data.columns.astype(str).str.strip() - mfp_data = mfp_data.loc[ - :, ~mfp_data.columns.str.startswith("Unnamed") & (mfp_data.columns != "") - ] + junk_col_pattern = r"^(Unnamed:.*||\.\d+)$" + mfp_data = mfp_data.loc[:, ~mfp_data.columns.str.match(junk_col_pattern)] - expected_columns = { + expected_columns = [ "Station", "Type", "Latitude", @@ -154,28 +135,25 @@ def _validate_mfp_data(file_path): "Distance to Next (NM)", "Ship Speed (kn)", "EEZ", - } - - actual_columns = set(mfp_data.columns) + ] + expected_set = set(expected_columns) + actual_set = set(mfp_data.columns) - missing_columns = expected_columns - actual_columns + missing_columns = expected_set - actual_set if missing_columns: raise ValueError( - f"Error: Found columns {list(actual_columns)}, but expected columns {list(expected_columns)}. " - "Are you sure that you're using the correct export from MFP?\n\n" - + errmsg_supplement + f"Error: Found columns {list(actual_set)}, but expected columns {list(expected_columns)}. " + f"Are you sure that you're using the correct export from MFP?\n\n{ERR_SUPPLEMENT}" ) - extra_columns = actual_columns - expected_columns + extra_columns = actual_set - expected_set if extra_columns: - # TODO: as mentioned below, propagate this warning to the user via the click.echo() output in the `virtualship init` command? warnings.warn( - f"Found additional unexpected columns {list(extra_columns)}. " - "Manually added columns have no effect. " + errmsg_supplement, + f"Found additional unexpected columns {list(extra_columns)}. Manually added columns have no effect.", stacklevel=2, ) - # Convert latitude and longitude to floats, handling commas and missing values safely + # safe float conversion for lat/lon for coord in ["Latitude", "Longitude"]: if mfp_data[coord].dtype in ["object", "string"]: mfp_data[coord] = pd.to_numeric( @@ -192,14 +170,12 @@ def _validate_mfp_data(file_path): "Arrival Port" in mfp_data["Station"].values or "Arrival Port" in mfp_data["Type"].values ) + if not has_departure or not has_arrival: - # TODO: propagate the warning to to the user via the click.echo() output in the `virtualship init` command, so that the user sees clearly it in the terminal. Perhaps a warnings section at the bottom. warnings.warn( "The MFP export is missing either a 'Departure Port' or 'Arrival Port', or both. " - "Any missing port will be replaced with a placeholder in `expedition.yaml` but will be ignored in the simulation. " - "The prescribed date will be used for Waypoint #1 instead. " - "If you believe this warning is wrong (i.e. you have selected departure/arrival ports), and " - + errmsg_supplement.replace("If ", ""), + "Any missing port will be replaced with an empty placeholder in `expedition.yaml` but will be ignored in the simulation. " + "If missing the 'Departure Port', the prescribed start date will be used for Waypoint #1 instead. ", stacklevel=2, ) @@ -216,10 +192,10 @@ def _validate_mfp_data(file_path): # convert 'Travel Time to Next' and 'Time at Station' to timedelta mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( - lambda x: _mfp_string_to_timedelta(x) + _mfp_string_to_timedelta ) mfp_data["Time at Station"] = mfp_data["Time at Station"].apply( - lambda x: _mfp_string_to_timedelta(x) + _mfp_string_to_timedelta ) # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column @@ -231,14 +207,12 @@ def _validate_mfp_data(file_path): return mfp_data -def _load_mfp_export(file_path): +def _load_mfp_export(file_path: Path) -> pd.DataFrame: if not os.path.isfile(file_path): raise FileNotFoundError(f"File not found: {file_path}") try: - df = pd.read_excel(file_path) - return df.dropna(how="all", axis=1) # drop empty columns - + return pd.read_excel(file_path).dropna(how="all", axis=1) # drop empty columns except Exception as e: raise RuntimeError( "Could not read coordinates data from the provided file. " @@ -246,7 +220,7 @@ def _load_mfp_export(file_path): ) from e -def _create_port_row(columns, port_type): +def _create_port_row(columns, port_type: str) -> pd.DataFrame: """Generate a single placeholder row for missing departure/arrival ports.""" row = {col: None for col in columns} row["Station"] = port_type @@ -254,14 +228,20 @@ def _create_port_row(columns, port_type): return pd.DataFrame([row]) -def _mfp_string_to_timedelta(value: str) -> timedelta: - """Handle MFP export string format (e.g., "0d 13h 13m").""" - if pd.isna(value): # last waypoint/missing ports have NaN/None travel time - return value # return None +def _mfp_string_to_timedelta(value: str | None) -> timedelta | None: + """Parse MFP duration string (e.g., '0d 13h 13m') to timedelta.""" + if pd.isna(value): + return None + + match = re.search(r"(\d+)d\s*(\d+)h\s*(\d+)m", str(value)) + if match: + days, hours, minutes = map(int, match.groups()) + return timedelta(days=days, hours=hours, minutes=minutes) - value = value.replace("d", ":").replace("h", ":").replace("m", "") - days, hours, minutes = map(int, value.split(":")) - return timedelta(days=days, hours=hours, minutes=minutes) + else: + raise ValueError( + f"Invalid MFP duration format: '{value}'. Expected format: 'Xd Yh Zm' (e.g., '0d 13h 13m'). {ERR_SUPPLEMENT}" + ) def _load_static_file(name: str) -> str: @@ -269,8 +249,7 @@ def _load_static_file(name: str) -> str: return files("virtualship.static").joinpath(name).read_text(encoding="utf-8") -@lru_cache(None) -@lru_cache(None) +@lru_cache(maxsize=1) def _get_example_expedition() -> str: """Get the example unified expedition configuration file.""" return _load_static_file(EXPEDITION)