feat!: PreTab 1.0.0 restructure and refactoring

.github/workflows/ci.yml

@@ -0,0 +1,279 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ci-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-lint-${{ runner.os }}-3.10-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run ruff check
+        run: poetry run ruff check .
+
+      - name: Run ruff format check
+        run: poetry run ruff format --check .
+
+  typecheck:
+    name: Type check (pyright)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-typecheck-${{ runner.os }}-3.10-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run pyright
+        run: poetry run pyright
+
+  build:
+    name: Build package
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-build-${{ runner.os }}-3.10-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install --only main
+
+      - name: Build package
+        run: poetry build
+
+      - name: Check package (twine)
+        run: |
+          python -m pip install --upgrade twine
+          twine check dist/*
+
+  tests:
+    name: Tests (Python ${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-tests-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run unit tests
+        run: poetry run pytest tests/ -v
+
+  smoke:
+    name: Smoke tests (Python 3.12, ubuntu)
+    runs-on: ubuntu-latest
+    needs: lint
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-smoke-${{ runner.os }}-3.12-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run smoke tests
+        run: poetry run pytest tests/ -v -m smoke --tb=short
+
+      - name: Run quickstart
+        run: poetry run python scripts/quickstart.py
+
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-latest
+    needs: tests
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-coverage-${{ runner.os }}-3.12-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Run tests with coverage
+        run: |
+          poetry run pytest tests/ \
+            --cov=pretab \
+            --cov-branch \
+            --cov-report=term-missing \
+            --cov-report=xml:coverage.xml \
+            --cov-fail-under=90 \
+            -q
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.xml
+          retention-days: 30
+
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
+
+  optional-deps:
+    name: Optional deps (${{ matrix.extra }})
+    runs-on: ubuntu-latest
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - extra: embeddings
+            module: sentence_transformers
+          - extra: lightgbm
+            module: lightgbm
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Configure Poetry
+        run: poetry config virtualenvs.in-project true
+
+      - name: Cache virtualenv
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-optdeps-${{ matrix.extra }}-${{ runner.os }}-3.12-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies with the ${{ matrix.extra }} extra
+        run: poetry install --extras "${{ matrix.extra }}"
+
+      - name: Verify the optional dependency imports
+        run: poetry run python -c "import ${{ matrix.module }}; print('${{ matrix.module }} import OK')"
+
+      - name: Run the suite with the extra installed
+        run: poetry run pytest tests/ -q

.github/workflows/docs.yml

@@ -50,6 +50,11 @@ jobs:
       - name: Install package and docs dependencies
         run: poetry install --with docs
 
+      # pandoc is required once the P14.1 notebook tutorials (nbsphinx / myst-nb)
+      # land; installing it now keeps the strict build forward-compatible.
+      - name: Install pandoc
+        run: sudo apt-get update && sudo apt-get install -y pandoc
+
       - name: Build Sphinx docs
         run: poetry run sphinx-build -b html docs docs/_build/html -W --keep-going
 

.gitignore

@@ -185,4 +185,4 @@ post-commit
 post-merge
 pre-push
 docs/notebooks/*
-docs/notebooks/
+docs/notebooks/

CHANGELOG.md

@@ -7,8 +7,6 @@ This project adheres to [Semantic Versioning](https://semver.org/) and uses
 
 Going forward, this file is updated automatically by `cz bump` on each release.
 
----
-
 ## Unreleased
 
 > **Note:** `0.1.0` is an internal development marker for the pre-1.0 restructure
@@ -20,6 +18,17 @@ Going forward, this file is updated automatically by `cz bump` on each release.
 
 ### Feat
 
+- **extension**: add a public, discoverable extension protocol: a `BaseRepresentation` base class (declaring `representation_name` / `feature_kind` / `scope` / `supervision`) that inherits the shared scikit-learn contract; `register_representation(name, cls)` plus opt-in `load_entry_point_representations()` (the `pretab.representations` entry-point group) to add third-party methods so they are selectable via `Preprocessor(numerical_method=...)`; `list_representations(feature_kind=, scope=, supervised=, periodic=, sparse_output=, adaptive=)` capability discovery; and a `check_representation(cls)` conformance suite (raising the new `RepresentationConformanceError`) verifying fit-returns-self, no input mutation, stable shape, matching feature names, determinism, fitted-state checks, and declared scope/supervision. `Preprocessor` gains transparent `preset="standard"|"expanded"|"adaptive"` aliases and `get_resolved_config()`; `TransformerSpec` gains `periodic` / `sparse_output` capability flags. A runnable sibling example lives at `examples/pretab-chebyshev` (all new symbols exported from `pretab`)
+- **serialize**: add portable, versioned serialization to `Preprocessor` (`to_spec` / `from_spec`) that captures a fitted preprocessor as a schema- and dependency-versioned JSON document and reconstructs it bit-for-bit, an auditable, allow-listed alternative to `pickle` that never executes estimator code on load; add a stable cross-process `fingerprint_` (sha256 over the resolved config, seeds, versions, output order, and fitted state) with a `reproducibility_report()`; and add an immutable lifecycle (`lifecycle_state_` ∈ `UNFITTED` / `FITTED` / `FROZEN` / `STALE`, `freeze` / `is_frozen` / `mark_stale` / `clone_unfitted` / `refit`) where `set_params` on a frozen preprocessor raises the new `PretabSerializationError` / `FrozenRepresentationError` (both exported from `pretab`)
+- **missing**: add a high-level `Preprocessor(missing_policy=...)` control (`error` / `propagate` / `impute` / `impute_with_indicator` / `separate_state`) that overrides the low-level imputation parameters; `separate_state` emits a dedicated `__missing` column (new `MissingStateIndicator`, wired through a per-column `FeatureUnion`) that stays outside the ordinary representation basis, and `error` rejects missing input at fit/transform; pin the end-to-end edge-case behaviour (constant features, `custombin` determinism, duplicate support points, missing values, unseen categories) in `tests/regression/test_edge_cases.py`
+- **output**: add output-budget controls to `Preprocessor` (`max_output_features`, `max_features_per_input`, `max_dense_memory`, `overflow_policy`, plus `estimate_output_shape` / `estimate_memory`, raising the new `OutputBudgetError`) and first-class output-format control (`output_format ∈ {auto, dense, sparse}`, `dtype`, an `output_report_` memory report, and `set_output(transform="pandas"|"polars")` DataFrame wrapping); defaults (`dense`, no budgets) reproduce historical behaviour
+- **policy**: add a central `RepresentationPolicy(missing, constant, out_of_range, invalid)` (exported from `pretab`) and a `Preprocessor(policy=...)` hook (resolved to `policy_` at fit) governing constant-column, out-of-range, and non-finite handling; defaults reproduce historical behaviour. Pin the per-family edge-case contract (constant column, all-missing, partial-missing propagation, tiny n, duplicate support points, out-of-range, infinity, feature-count mismatch) in `tests/test_edge_case_contract.py`, and fix silent-corruption gaps so every spline family raises a typed `PretabDataError` on a constant or all-missing column (and cleanly propagates partial-missing rows), feature maps reject all-missing columns, and `NumericBinningTransformer` rejects non-finite input
+- **supervised**: add a leakage-safe supervised contract: `requires_y` / `is_supervised` / fitted `uses_target_` on every transformer, a `LeakageWarning` when a target-aware transformer is fit on `(X, y)` outside a Pipeline / cross-validation context, a `CrossFittedTransformer` wrapper that produces out-of-fold training features (recording `cross_fitted` / `n_folds` in the spec), and a `RepresentationSearchCV` skeleton (all exported from `pretab`)
+- **representation**: add typed `RepresentationSpec` and per-output-column `FeatureLineage` (exported from `pretab`); every transformer family exposes `get_representation_spec()` and `Preprocessor.get_feature_lineage()` maps each output column to its source feature(s), representation family, component, and target-usage flag
+- **transformers**: add `FourierFeatureTransformer` (deterministic sine/cosine feature map with `harmonic` / `log_spaced` / `random` frequencies), selectable as the `"fourier"` numerical method
+- **transformers**: add `RandomFourierFeaturesTransformer` and `NystroemFeaturesTransformer`, standalone multivariate kernel-approximation feature maps (`"rff"` / `"nystroem"`)
+- **binning**: make `NumericBinningTransformer` a stateful, multi-feature encoder with learned `bin_edges_` and `encode` (`ordinal` / `onehot` / `soft`) plus `placement_strategy` (`uniform` / `quantile`) options
+- **transformers**: add `harmonics` and `include_original` options to `PeriodicEncodingTransformer` for multi-harmonic periodic encodings
 - update default output_dim
 - unsupervised feature-map default
 - wire custombin output_dim
@@ -28,7 +37,8 @@ Going forward, this file is updated automatically by `cz bump` on each release.
 - **pipeline**: use selector and adaptive setting to splines
 - **pipeline**: accept preprocessing method name variations
 - **preprocessor**: expose total_output_dim_, output_dims_ attribute
-- **preprocessor**: add random_state, handle_missing parameters
+- **preprocessor**: add random_state parameter
+- **preprocessor**: add numerical_imputation / categorical_imputation / add_missing_indicator parameters (replacing handle_missing)
 - **sklearn-compat**: enforce n_features consistency, fix mixin order/tags
 - **exceptions**: route all raises through typed exceptions
 - **logging**: add verbose level, route warnings
@@ -69,6 +79,13 @@ Going forward, this file is updated automatically by `cz bump` on each release.
 
 ### Refactor
 
+- **preprocessor**: collapse the duplicated feature name in `Preprocessor` output column names (`get_feature_names_out()`, `return_array=True`, `set_output(transform="pandas"|"polars")`, and `get_feature_lineage()`); a column previously named `num_annual_income__annual_income_ncs0` is now `num_annual_income_ncs0`. Dict-mode output keys (`num_<col>` / `cat_<col>`) and standalone transformer usage outside `Preprocessor` are unaffected
+- **splines**: reformulate `ThinPlateSplineTransformer` as a multivariate low-rank thin-plate regression spline (landmark selection + eigen/Nyström basis via `n_components` / `landmark_strategy` / `rank_strategy`, replacing the univariate `output_dim` form)
+- **transformers**: rename `CustomBinTransformer` → `NumericBinningTransformer`, `CyclicalTimeTransformer` → `PeriodicEncodingTransformer`, and `CubicSplineTransformer` → `CubicRegressionSplineTransformer` (intention-revealing public names)
+- **transformers**: remove `LagFeatureTransformer` and `RollingStatsTransformer` (row-count-changing time-series utilities outside the tabular scope)
+- **splines**: restrict `PSplineTransformer` to `placement_strategy="uniform"` (penalized splines require equally-spaced knots)
+- **compose**: exclude the multivariate `tensorspline` / `tprs` methods from the per-column `Preprocessor` whitelist (they remain available as standalone transformers)
+- **categorical**: deprecate `OneHotFromOrdinalTransformer` (use the `"one-hot"` categorical method backed by scikit-learn's `OneHotEncoder`)
 - consistent param order
 - remove dead selection helpers
 - **ple**: use location selectors for thresholds
@@ -97,3 +114,7 @@ Going forward, this file is updated automatically by `cz bump` on each release.
 - Adopted a Poetry + OIDC release pipeline publishing to PyPI (`v*.*.*`) and TestPyPI (`v*.*.*rc*`), plus a manual `build-check` dry-run workflow
 - Added a `justfile` and pre-commit configuration for the local development workflow
 - Added project meta documentation: `CHANGELOG.md`, `CONVENTIONAL_COMMITS.md`, and `CODE_OF_CONDUCT.md`
+- Drove `pyright` to zero errors across the package and test suite and promoted the CI `typecheck` job from advisory to required
+- Hardened `ci.yml` with an `optional-deps` job that installs the `embeddings` and `lightgbm` extras and runs the suite against each, and wired a `--cov-fail-under=90` gate into the coverage job
+- Added `scripts/quickstart.py`, a runnable, CI-gated smoke test covering mixed-type preprocessing, feature lineage, leakage-safe cross-fitting, sklearn `Pipeline` compatibility, serialization round-trips, and representation discovery (`just quickstart`)
+- Added root `CONTRIBUTING.md` and `SECURITY.md` so GitHub surfaces the contributor guide and a private vulnerability-reporting channel

CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing
+
+Thanks for your interest in contributing to PreTab.
+
+The full contributor guide, covering environment setup, the local development
+workflow, and what a pull request needs to pass review, lives in the
+documentation:
+
+**[Contributing Guide](https://pretab.readthedocs.io/en/latest/developer_guide/contributing.html)**
+
+Quick start:
+
+```bash
+git clone https://github.com/OpenTabular/PreTab
+cd PreTab
+just install
+just test
+just check
+```
+
+All contributors are expected to follow the [Code of Conduct](CODE_OF_CONDUCT.md).
+
+> **Note:** Report bugs and request features on the
+> [issue tracker](https://github.com/OpenTabular/PreTab/issues). Security
+> vulnerabilities should be reported privately; see [SECURITY.md](SECURITY.md).