Support multiple pins and pin+data frame tables via a shared DuckDB executor

pkg-py/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### New features
+
+* Multiple pins (and pins mixed with data frames) are now supported in one chat: pin and data-frame tables are materialized into a shared DuckDB connection, so the LLM can join and filter across them. Each pin still keeps its own private connection for standalone use. (#312)
+
 ### Changes
 
 * `.server(data_source=)` no longer modifies the `QueryChat` instance. The table is registered for that session only: the instance's tables, greeting tables, and system prompt are unchanged, a same-named instance table is shadowed for that session, and the session's data source is cleaned up when the session ends. This removes the concurrent-session edge cases that `0.8.0` patched around (#300, #302, #303, #304, #308).
@@ -21,8 +25,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * `cleanup()` no longer closes a spec-resolved `.server(client=...)` override while its session is still running; it is closed when the session ends.
 
-* Registering a second pins table with `add_table()` now raises a clear error at registration time instead of failing at query time: each pin queries through its own DuckDB connection, so only the first pin's table would be queryable. To combine a pin with other tables, register them in a shared DuckDB connection and pass that instead.
-
 ## [0.8.0] - 2026-09-12
 
 ### New features

pkg-py/src/querychat/_datasource.py

@@ -117,6 +117,11 @@ def duckdb_column_meta(name: str, duckdb_type: Any) -> ColumnMeta:
     return ColumnMeta(name=name, sql_type=sql_type, kind=kind)
 
 
+def quote_identifier(name: str) -> str:
+    """Return ``name`` as a double-quoted SQL identifier."""
+    return '"' + name.replace('"', '""') + '"'
+
+
 def duckdb_column_stats(
     conn: duckdb.DuckDBPyConnection,
     table_name: str,
@@ -137,7 +142,9 @@ def duckdb_column_stats(
         return
 
     try:
-        stats_query = f'SELECT {", ".join(select_parts)} FROM "{table_name}"'
+        stats_query = (
+            f"SELECT {', '.join(select_parts)} FROM {quote_identifier(table_name)}"
+        )
         result = conn.execute(stats_query).fetchone()
         if not result:
             return
@@ -162,7 +169,7 @@ def duckdb_column_stats(
     try:
         for col in categorical_cols:
             cat_result = conn.execute(
-                f'SELECT DISTINCT "{col.name}" FROM "{table_name}" '
+                f'SELECT DISTINCT "{col.name}" FROM {quote_identifier(table_name)} '
                 f'WHERE "{col.name}" IS NOT NULL ORDER BY "{col.name}"'
             ).fetchall()
             col.categories = [str(row[0]) for row in cat_result]
@@ -501,6 +508,18 @@ def get_data(self) -> IntoDataFrameT:
         """
         return self._df.to_native()
 
+    def register_into(
+        self, conn: duckdb.DuckDBPyConnection, table_name: str | None = None
+    ) -> None:
+        """
+        Register this DataFrame in a shared DuckDB connection.
+
+        Internal hook for joining a shared DuckDB executor. The caller owns
+        ``conn`` and locks it down once all tables are registered.
+        """
+        # NOTE: if native representation is polars, pyarrow is required for registration
+        conn.register(table_name or self.table_name, self.get_data())
+
     def cleanup(self) -> None:
         """
         Close the DuckDB connection.

pkg-py/src/querychat/_pin_source.py

@@ -2,6 +2,7 @@
 
 import re
 from typing import TYPE_CHECKING, Any, TypeGuard
+from uuid import uuid4
 
 import duckdb
 import narwhals.stable.v1 as nw
@@ -14,6 +15,7 @@
     duckdb_column_stats,
     duckdb_lock_down,
     format_schema,
+    quote_identifier,
 )
 from ._utils import check_query
 
@@ -65,6 +67,21 @@ def _convert_result(result: duckdb.DuckDBPyConnection) -> nw.DataFrame:
     return nw.from_native(result.df())
 
 
+def stage_frame_as_table(
+    conn: duckdb.DuckDBPyConnection, frame: Any, table_name: str
+) -> None:
+    """Materialize a DataFrame as a real table via a unique staging view."""
+    vname = f"__pin_staging_{table_name}_{uuid4().hex[:8]}"
+    conn.register(vname, frame)
+    try:
+        conn.execute(
+            f"CREATE TABLE {quote_identifier(table_name)} AS "
+            f"SELECT * FROM {quote_identifier(vname)}"
+        )
+    finally:
+        conn.unregister(vname)
+
+
 class PinSource(DataSource[nw.DataFrame]):
     """
     DataSource backed by a pin from a pins board.
@@ -80,6 +97,13 @@ class PinSource(DataSource[nw.DataFrame]):
     :class:`~querychat.QueryChat` uses them as the default
     ``data_description``, which you can override.
 
+    Multiple pins
+    ~~~~~~~~~~~~~
+
+    Multiple pins (and pins mixed with data frames) can be combined in one
+    chat: every table is materialized into a shared DuckDB connection, so
+    the LLM can join and filter across them.
+
     Lazy queries with pins
     ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -118,61 +142,24 @@ def __init__(
         effective_table_name = _sanitize_table_name(table_name or name)
         self.table_name = effective_table_name
 
+        # Retained so the pin can be re-materialized into a shared DuckDB
+        # connection when it joins a multi-table executor (register_into()).
+        self._board = board
+        self._pin_name = name
+
         self._pin_meta_obj = board.pin_meta(name, version=version)
-        pin_type = self._pin_meta_obj.type
+        # Snapshot the resolved version so register_into() reads the same pin
+        # content even if the pin is updated after construction. meta.version
+        # is a Version/VersionRaw (unwrap .version) or a plain string on some
+        # boards (e.g. Connect GUIDs).
+        resolved = getattr(self._pin_meta_obj.version, "version", None)
+        if not isinstance(resolved, str):
+            resolved = self._pin_meta_obj.version
+        self._version: str | None = resolved if isinstance(resolved, str) else version
 
         conn = duckdb.connect()
         try:
-            if pin_type in DUCKDB_FILE_TYPES:
-                paths = board.pin_download(name, version=version)
-                if len(paths) != 1:
-                    raise ValueError(
-                        f"Pin '{name}' contains {len(paths)} files, but PinSource "
-                        "requires a single-file pin (as created by pin_write())."
-                    )
-                reader_fn = DUCKDB_READER_FN[pin_type]
-                if pin_type == "json":
-                    conn.execute("INSTALL json")
-                    conn.execute("LOAD json")
-                conn.execute(
-                    f'CREATE TABLE "{effective_table_name}" AS '
-                    f"SELECT * FROM {reader_fn}(?)",
-                    [paths[0]],
-                )
-            elif pin_type == "arrow" and _has_polars():
-                # Arrow/IPC files can't be read natively by DuckDB, but
-                # polars can read them directly — avoiding pin_read() overhead.
-                import polars as pl
-
-                paths = board.pin_download(name, version=version)
-                if len(paths) != 1:
-                    raise ValueError(
-                        f"Pin '{name}' contains {len(paths)} files, but PinSource "
-                        "requires a single-file pin (as created by pin_write())."
-                    )
-                arrow_df = pl.read_ipc(paths[0])
-                vname = f"__pin_staging_{effective_table_name}"
-                conn.register(vname, arrow_df)
-                conn.execute(
-                    f'CREATE TABLE "{effective_table_name}" AS SELECT * FROM "{vname}"'
-                )
-                conn.unregister(vname)
-            else:
-                import pandas as pd
-
-                data = board.pin_read(name, version=version)
-                if not isinstance(data, pd.DataFrame):
-                    raise TypeError(
-                        f"Pin '{name}' contains {type(data).__name__}, not a DataFrame. "
-                        "PinSource requires the pin to contain a pandas DataFrame."
-                    )
-                vname = f"__pin_staging_{effective_table_name}"
-                conn.register(vname, data)
-                conn.execute(
-                    f'CREATE TABLE "{effective_table_name}" AS SELECT * FROM "{vname}"'
-                )
-                conn.unregister(vname)
-
+            self._materialize_into(conn, effective_table_name)
             duckdb_lock_down(conn)
         except Exception:
             conn.close()
@@ -184,6 +171,80 @@ def __init__(
         result = self._conn.execute(f'SELECT * FROM "{effective_table_name}" LIMIT 0')
         self._colnames = [desc[0] for desc in result.description]
 
+    def _materialize_into(self, conn: duckdb.DuckDBPyConnection, table_name: str):
+        """
+        Materialize the pin as a real table in ``conn``.
+
+        Does not lock the connection down; the caller owns ``conn`` and
+        decides when (or whether) to call :func:`duckdb_lock_down`.
+        """
+        board, name, version = self._board, self._pin_name, self._version
+        pin_type = self._pin_meta_obj.type
+
+        if pin_type in DUCKDB_FILE_TYPES:
+            paths = board.pin_download(name, version=version)
+            if len(paths) != 1:
+                raise ValueError(
+                    f"Pin '{name}' contains {len(paths)} files, but PinSource "
+                    "requires a single-file pin (as created by pin_write())."
+                )
+            reader_fn = DUCKDB_READER_FN[pin_type]
+            if pin_type == "json":
+                conn.execute("INSTALL json")
+                conn.execute("LOAD json")
+            conn.execute(
+                f"CREATE TABLE {quote_identifier(table_name)} AS "
+                f"SELECT * FROM {reader_fn}(?)",
+                [paths[0]],
+            )
+        elif pin_type == "arrow" and _has_polars():
+            # Arrow/IPC files can't be read natively by DuckDB, but
+            # polars can read them directly — avoiding pin_read() overhead.
+            import polars as pl
+
+            paths = board.pin_download(name, version=version)
+            if len(paths) != 1:
+                raise ValueError(
+                    f"Pin '{name}' contains {len(paths)} files, but PinSource "
+                    "requires a single-file pin (as created by pin_write())."
+                )
+            arrow_df = pl.read_ipc(paths[0])
+            stage_frame_as_table(conn, arrow_df, table_name)
+        else:
+            import pandas as pd
+
+            data = board.pin_read(name, version=version)
+            if not isinstance(data, pd.DataFrame):
+                raise TypeError(
+                    f"Pin '{name}' contains {type(data).__name__}, not a DataFrame. "
+                    "PinSource requires the pin to contain a pandas DataFrame."
+                )
+            stage_frame_as_table(conn, data, table_name)
+
+    def register_into(
+        self, conn: duckdb.DuckDBPyConnection, table_name: str | None = None
+    ) -> None:
+        """
+        Materialize this pin into a shared DuckDB connection.
+
+        Internal hook for joining a shared DuckDB executor (multiple pins, or
+        pins mixed with data frames). The caller owns ``conn`` and locks it
+        down once all tables are materialized.
+        """
+        target = table_name or self.table_name
+        from pins.errors import PinsError
+
+        try:
+            self._materialize_into(conn, target)
+        except PinsError as e:
+            # The snapshotted pin version may have been pruned (e.g. a
+            # non-versioned board rewritten after construction); fall back to
+            # this source's own copy so the shared table matches the private
+            # connection. Other materialization failures still raise.
+            if "missing version" not in str(e):
+                raise
+            stage_frame_as_table(conn, self.get_data().to_native(), target)
+
     def get_db_type(self) -> str:
         return "DuckDB"