Skip to content

Commit e68a1b8

Browse files
fix(populate): pin parallel populate to the fork start method
The real cause of the py3.14 failure is Python 3.14 changing the multiprocessing default start method from fork to forkserver. Parallel populate hands the live table and its open DB connection to workers by process inheritance (fork); under forkserver the payload is pickled instead, which cannot carry a live connection and deadlocked the job (after first surfacing as 'cannot pickle itertools.count'). Pin both Pool call sites to a fork context (fork where available, platform default otherwise). Reverts the earlier Dependencies __getstate__/__setstate__ pickle workaround, which addressed only the symptom. py3.10 (fork default) already passed; this makes py3.14 use the same proven path.
1 parent 605f833 commit e68a1b8

2 files changed

Lines changed: 17 additions & 19 deletions

File tree

src/datajoint/autopopulate.py

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@
2222

2323
logger = logging.getLogger(__name__.split(".")[0])
2424

25+
# Parallel populate hands the live table and its open connection to workers by
26+
# process inheritance, so it requires the `fork` start method. Python 3.14
27+
# changed the default from `fork` to `forkserver`, which pickles the payload
28+
# instead — that neither carries a live DB connection nor the table's internal
29+
# state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
30+
# fall back to the platform default elsewhere.
31+
_MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
32+
2533

2634
# --- helper functions for multiprocessing --
2735

@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
3038
"""
3139
Initialize a worker process for multiprocessing.
3240
33-
Saves the unpickled table to the current process and reconnects to database.
41+
Stores the inherited table on the worker process and reconnects to database
42+
(the parent closes its connection before forking; each worker reopens one).
3443
3544
Parameters
3645
----------
@@ -476,7 +485,9 @@ def _populate_direct(
476485
if hasattr(self.connection._conn, "ctx"):
477486
del self.connection._conn.ctx
478487
with (
479-
mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool,
488+
mp.get_context(_MP_START_METHOD).Pool(
489+
processes, _initialize_populate, (self, None, populate_kwargs)
490+
) as pool,
480491
tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
481492
):
482493
for status in pool.imap(_call_populate1, keys, chunksize=1):
@@ -572,7 +583,9 @@ def handler(signum, frame):
572583
if hasattr(self.connection._conn, "ctx"):
573584
del self.connection._conn.ctx # SSLContext is not pickleable
574585
with (
575-
mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool,
586+
mp.get_context(_MP_START_METHOD).Pool(
587+
processes, _initialize_populate, (self, self.jobs, populate_kwargs)
588+
) as pool,
576589
tqdm(desc="Processes: ", total=nkeys)
577590
if display_progress
578591
else contextlib.nullcontext() as progress_bar,
@@ -866,8 +879,6 @@ def _update_job_metadata(self, key, start_time, duration, version):
866879

867880
pk_condition = make_condition(self, key, set())
868881
self.connection.query(
869-
f"UPDATE {self.full_table_name} SET "
870-
"_job_start_time=%s, _job_duration=%s, _job_version=%s "
871-
f"WHERE {pk_condition}",
882+
f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
872883
args=(start_time, duration, version[:64] if version else ""),
873884
)

src/datajoint/dependencies.py

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -140,19 +140,6 @@ def clear(self) -> None:
140140
self._node_alias_count = itertools.count() # reset alias IDs for consistency
141141
super().clear()
142142

143-
def __getstate__(self) -> dict:
144-
# itertools.count is not picklable. Multiprocessing populate pickles the
145-
# table (and thus its connection's Dependencies) to worker processes,
146-
# where dependencies are reloaded — so the counter is dropped here and
147-
# rebuilt in __setstate__ rather than carried across the pickle.
148-
state = self.__dict__.copy()
149-
state.pop("_node_alias_count", None)
150-
return state
151-
152-
def __setstate__(self, state: dict) -> None:
153-
self.__dict__.update(state)
154-
self._node_alias_count = itertools.count()
155-
156143
def load(self, force: bool = True, schema_names: set[str] | None = None) -> None:
157144
"""
158145
Load dependencies for the given schemas.

0 commit comments

Comments
 (0)