Merge pull request #1489 from datajoint/retire/strict-provenance

Retire strict_provenance runtime guardrail (#1488)

Author: dimitri-yatsenko

README.md

@@ -5,7 +5,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
 - **Tables represent workflow steps** — Each table is a step in your pipeline
 - **Foreign keys encode dependencies** — Parent tables must be populated before child tables
 - **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
-- **Results are immutable** — Full provenance and reproducibility
+- **Results are immutable** — Full lineage and reproducibility
 
 **Documentation:** https://docs.datajoint.com
 

src/datajoint/autopopulate.py

@@ -111,10 +111,11 @@ def upstream(self):
         (or ``FreeTable``, when indexed by a string) for any ancestor of
         ``self``.
 
-        Reading via ``self.upstream`` is the provenance-safe pattern: the
-        framework guarantees the restriction matches the current ``key``,
-        and indexing a non-ancestor table raises ``DataJointError``. See
-        :doc:`reference/specs/provenance` for the contract.
+        Reading via ``self.upstream`` is the recommended pattern for the
+        make() reproducibility contract: the framework guarantees the
+        restriction matches the current ``key``, and indexing a non-ancestor
+        table raises ``DataJointError``. See
+        :doc:`reference/specs/autopopulate` for the contract.
 
         Raises
         ------
@@ -671,34 +672,6 @@ def _populate1(
 
         self._upstream = Diagram.trace(self & dict(key))
 
-        # If strict_provenance is on, push the active-make context so the
-        # runtime gates in expression.cursor / table.insert can check this
-        # make()'s reads and writes. The context is popped in the finally
-        # block below.
-        strict_token = None
-        if self.connection._config.get("strict_provenance", False):
-            from .provenance import push_strict_make_context
-            from .user_tables import Part
-
-            allowed_tables = set(self._upstream._cascade_restrictions.keys()) | {self.full_table_name}
-            # Add Part tables of self to the allowed set. Use class __dict__
-            # (not dir/getattr) to avoid triggering descriptors like the
-            # _JobsDescriptor that lazy-declares the ~~ job table.
-            for cls in type(self).__mro__:
-                for attr_name, attr in cls.__dict__.items():
-                    if attr_name.startswith("_"):
-                        continue
-                    if isinstance(attr, type) and issubclass(attr, Part):
-                        # Instantiate to get full_table_name resolved against
-                        # this schema. The Part class is already attached via
-                        # @schema decoration of the master.
-                        try:
-                            part_ftn = attr().full_table_name
-                            allowed_tables.add(part_ftn)
-                        except Exception:
-                            pass
-            strict_token = push_strict_make_context(self, frozenset(allowed_tables), dict(key))
-
         try:
             if not is_generator:
                 make(dict(key), **(make_kwargs or {}))
@@ -760,11 +733,6 @@ def _populate1(
             # access raises a clear error rather than silently using a
             # stale trace from the previous make() call.
             self._upstream = None
-            # Pop the strict-make context, if any.
-            if strict_token is not None:
-                from .provenance import pop_strict_make_context
-
-                pop_strict_make_context(strict_token)
 
     def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
         """

src/datajoint/expression.py

@@ -1242,12 +1242,6 @@ def cursor(self, as_dict=False):
         cursor
             Database query cursor.
         """
-        # Strict-provenance read gate. No-op outside make() or when the
-        # config flag is off. See src/datajoint/provenance.py.
-        from .provenance import assert_read_allowed
-
-        assert_read_allowed(self)
-
         sql = self.make_sql()
         logger.debug(sql)
         return self.connection.query(sql, as_dict=as_dict)

src/datajoint/provenance.py

@@ -69,7 +69,6 @@
     "database.database_prefix": "DJ_DATABASE_PREFIX",
     "database.create_tables": "DJ_CREATE_TABLES",
     "loglevel": "DJ_LOG_LEVEL",
-    "strict_provenance": "DJ_STRICT_PROVENANCE",
     "display.diagram_direction": "DJ_DIAGRAM_DIRECTION",
 }
 
@@ -362,16 +361,6 @@ class Config(BaseSettings):
         "*New in 2.2.3.*",
     )
 
-    strict_provenance: bool = Field(
-        default=False,
-        validation_alias="DJ_STRICT_PROVENANCE",
-        description="If True, enforces the upstream-only convention inside make(): "
-        "reads must go through self.upstream[Ancestor], writes must target self "
-        "or self's Part tables with primary keys consistent with the current key. "
-        "Off by default; opt-in for deployments that need runtime provenance "
-        "guarantees backing downstream lineage / CDC tooling. *New in 2.3.*",
-    )
-
     # Cache path for query results
     query_cache: Path | None = None
 

src/datajoint/settings.py

@@ -834,23 +834,10 @@ def insert(
                 " To override, set keyword argument allow_direct_insert=True."
             )
 
-        # Strict-provenance write gate (target check only). No-op outside make()
-        # or when the config flag is off. Deliberately does NOT touch `rows` —
-        # the per-row key-consistency check happens in `_insert_rows` as rows are
-        # materialized, so a one-shot iterable (generator) is not consumed here.
-        # See src/datajoint/provenance.py.
-        from .provenance import assert_write_allowed
-
-        assert_write_allowed(self)
-
         if inspect.isclass(rows) and issubclass(rows, QueryExpression):
             rows = rows()  # instantiate if a class
         if isinstance(rows, QueryExpression):
             # insert from select - chunk_size not applicable.
-            # Note: this INSERT ... SELECT runs entirely server-side, so under
-            # strict_provenance the per-row key-consistency check does not apply
-            # (row values are never materialized client-side). The target check
-            # in assert_write_allowed above still governs which table is written.
             if chunk_size is not None:
                 raise DataJointError("chunk_size is not supported for QueryExpression inserts")
             if not ignore_extra_fields:
@@ -905,17 +892,7 @@ def _insert_rows(self, rows, replace, skip_duplicates, ignore_extra_fields):
         """
         # collects the field list from first row (passed by reference)
         field_list = []
-        # Strict-provenance per-row key check runs here, as each row is
-        # materialized — no-op outside make()/when the flag is off. Placing it in
-        # this single materialization point (reached by both the chunked and
-        # single-batch paths) avoids consuming the caller's `rows` iterable early.
-        from .provenance import assert_row_key_allowed
-
-        def _make_row(row):
-            assert_row_key_allowed(row)
-            return self.__make_row_to_insert(row, field_list, ignore_extra_fields)
-
-        rows = list(_make_row(row) for row in rows)
+        rows = list(self.__make_row_to_insert(row, field_list, ignore_extra_fields) for row in rows)
         if rows:
             try:
                 # Handle empty field_list (all-defaults insert)

src/datajoint/table.py

@@ -381,7 +381,7 @@ class Greeting(dj.Computed):
         """
 
         def make(self, key):
-            # Provenance-safe read: self.upstream pre-restricted to current key
+            # Upstream read: self.upstream pre-restricted to current key
             name = self.upstream[Subject].fetch1("name")
             self.insert1({**key, "greeting": f"Hello, {name}!"})