Debug library

.github/workflows/release.yml

@@ -64,6 +64,32 @@ jobs:
       - name: Build sdist and wheel
         run: python -m build
 
+      # Only one release/tag is ever kept per branch — delete whatever this
+      # branch's previous release+tag was (main: a plain "v*" tag; staging:
+      # a "v*-staging" tag) before creating the new one below. Best-effort:
+      # a delete failure (e.g. nothing to delete yet) must not fail the
+      # release itself.
+      - name: Delete this branch's previous release and tag
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set +e
+          if [ "${{ github.ref_name }}" = "staging" ]; then
+            SUFFIX="-staging"
+          else
+            SUFFIX=""
+          fi
+          for TAG in $(gh release list --limit 100 --json tagName -q '.[].tagName'); do
+            case "$TAG" in
+              *-staging) TAG_SUFFIX="-staging" ;;
+              *) TAG_SUFFIX="" ;;
+            esac
+            if [ "$TAG_SUFFIX" = "$SUFFIX" ]; then
+              echo "Deleting previous release/tag for ${{ github.ref_name }}: $TAG"
+              gh release delete "$TAG" --cleanup-tag -y
+            fi
+          done
+
       - name: Create GitHub release
         uses: softprops/action-gh-release@v3
         with:

README.md

@@ -19,10 +19,17 @@ Two main class domains for the EEA data lakehouse:
 
 ## Install
 
-Install directly from GitHub, pinned to a released tag:
+[![Latest release](https://img.shields.io/github/v/release/eeadata/EEALakeHouse.python?label=latest%20release)](https://github.com/eeadata/EEALakeHouse.python/releases/latest)
+
+The badge above always shows the current latest release tag — substitute it for `v0.1.5` below
+if it's moved on since this was written (or check the [Releases page](https://github.com/eeadata/EEALakeHouse.python/releases/latest) directly).
 
 ```bash
-pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.0"
+# latest release (currently v0.1.5) — recommended: stable, pinned to a tag
+pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.5"
+
+# latest main — bleeding edge, whatever's currently merged, not pinned to a release
+pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main"
 ```
 
 ## Usage
@@ -169,12 +176,23 @@ pytest
    Pushing to `main` without bumping the version re-publishes the release for
    the current version with the latest build artifacts.
 
+**Only one release/tag exists per branch at a time** — `main` always has exactly one `vX.Y.Z`
+release, `staging` exactly one `vX.Y.Z-staging` prerelease. Each new release deletes its branch's
+previous release *and* tag first, so **tags aren't permanent** — pin to whatever the
+[Install](#install) badge shows *now*, not to an old tag number, since it won't exist once a
+newer release replaces it.
+
 ## Install in JupyterLab
 
-Run this in a notebook cell, pinned to the release tag you want:
+Run this in a notebook cell — see the badge under [Install](#install) for the current latest
+release tag (`v0.1.5` as of this writing):
 
 ```python
-%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.0"
+# latest release (currently v0.1.5) — recommended
+%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.5"
+
+# latest main — bleeding edge, not pinned to a release
+%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main"
 ```
 
 Use the `%pip` magic rather than `!pip` — it installs into the kernel the
@@ -186,9 +204,9 @@ Kernel...**) so the import below picks up the newly installed package:
 from eea_datalakehouse.dds_ingestion import FolderIngest
 ```
 
-Alternatively, download the wheel attached to the GitHub Release page for
-that tag and install the local file instead of pulling from git:
+Alternatively, download the wheel attached to the [GitHub Release page](https://github.com/eeadata/EEALakeHouse.python/releases/latest)
+for that tag and install the local file instead of pulling from git:
 
 ```python
-%pip install /path/to/EEADataLakehouse-0.1.0-py3-none-any.whl
+%pip install /path/to/EEADataLakehouse-0.1.5-py3-none-any.whl
 ```

debugger/debug_run.py

@@ -27,6 +27,16 @@
 import os
 import json
 import httpx
+
+
+
+import argparse
+import atexit
+import subprocess
+import sys
+
+import requests
+
 from pathlib import Path
 from xmlrpc.client import Boolean
 from eea_datalakehouse.catalog import Catalog
@@ -38,7 +48,12 @@
 BASE_URL = "https://dds.debug.local"
 # This script always lives directly in debugger/, so its own directory *is*
 # the folder to look in — independent of whatever cwd the debugger/terminal
-# happened to launch with.
+# happened to launch with.   
+CACHE_PATH = os.path.expanduser("~/.dremio_msal_cache.json")
+TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange"
+JWT_TYPE = "urn:ietf:params:oauth:token-type:jwt"
+PAT_TYPE = "urn:ietf:params:oauth:token-type:dremio:personal-access-token"
+
 ENV_PATH = Path(__file__).resolve().parent / ".env"
 
 
@@ -346,6 +361,126 @@ def run_catalog_bulk_close() -> None:
     print(f"catalog_bulk_close  after   a.closed={a._closed}  b.closed={b._closed}")
 
 
+
+# --------------------------------------------------------------------------
+# Step 1 — get an Entra ID JWT
+# --------------------------------------------------------------------------
+def entra_token_azcli(scope: str) -> str:
+    """Reuse the Azure CLI's cached login. Silent, no prompts."""
+    out = subprocess.run(
+        ["az", "account", "get-access-token", "--scope", scope, "-o", "json"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(out.stdout)["accessToken"]
+
+
+def _msal_app(cls, **kwargs):
+    import msal
+
+    cache = msal.SerializableTokenCache()
+    if os.path.exists(CACHE_PATH):
+        cache.deserialize(open(CACHE_PATH).read())
+
+    def _flush():
+        if cache.has_state_changed:
+            fd = os.open(CACHE_PATH, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+            with os.fdopen(fd, "w") as fh:
+                fh.write(cache.serialize())
+
+    atexit.register(_flush)
+    return cls(token_cache=cache, **kwargs)
+
+
+def entra_token_device(tenant: str, client_id: str, scope: str) -> str:
+    """Device-code flow with a persistent cache: interactive once, silent after."""
+    import msal
+
+    app = _msal_app(
+        msal.PublicClientApplication,
+        client_id=client_id,
+        authority=f"https://login.microsoftonline.com/{tenant}",
+    )
+
+    accounts = app.get_accounts()
+    result = app.acquire_token_silent([scope], account=accounts[0]) if accounts else None
+
+    if not result:
+        flow = app.initiate_device_flow(scopes=[scope])
+        if "user_code" not in flow:
+            raise RuntimeError(f"device flow failed: {flow}")
+        print(flow["message"], file=sys.stderr)
+        result = app.acquire_token_by_device_flow(flow)
+
+    if "access_token" not in result:
+        raise RuntimeError(f"Entra ID error: {result.get('error_description', result)}")
+    return result["access_token"]
+
+
+def entra_token_sp(tenant: str, client_id: str, client_secret: str, scope: str) -> str:
+    """Client credentials — fully unattended, app identity."""
+    import msal
+
+    app = msal.ConfidentialClientApplication(
+        client_id=client_id,
+        client_credential=client_secret,
+        authority=f"https://login.microsoftonline.com/{tenant}",
+    )
+    result = app.acquire_token_for_client(scopes=[scope])
+    if "access_token" not in result:
+        raise RuntimeError(f"Entra ID error: {result.get('error_description', result)}")
+    return result["access_token"]
+
+
+# --------------------------------------------------------------------------
+# Step 2 — exchange the Entra JWT for a Dremio access token
+# --------------------------------------------------------------------------
+def dremio_exchange(host: str, subject_token: str, token_type: str = JWT_TYPE,
+                    verify: bool | str = True) -> dict:
+    r = requests.post(
+        f"https://{host}/oauth/token",
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+        data={
+            "grant_type": TOKEN_EXCHANGE,
+            "subject_token": subject_token,
+            "subject_token_type": token_type,
+            "scope": "dremio.all",
+        },
+        timeout=30,
+        verify=verify,
+    )
+    if not r.ok:
+        raise RuntimeError(f"Dremio token exchange failed [{r.status_code}]: {r.text}")
+    return r.json()
+
+
+# --------------------------------------------------------------------------
+# Step 3 — optional: mint a long-lived PAT with that access token
+# --------------------------------------------------------------------------
+def create_pat(host: str, access_token: str, username: str, label: str,
+               days: int = 90, verify: bool | str = True) -> dict:
+    h = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
+
+    u = requests.get(f"https://{host}/api/v3/user/by-name/{username}",
+                     headers=h, timeout=30, verify=verify)
+    if not u.ok:
+        raise RuntimeError(f"user lookup failed [{u.status_code}]: {u.text}")
+    user_id = u.json()["id"]
+
+    p = requests.post(
+        f"https://{host}/api/v3/user/{user_id}/token",
+        headers=h,
+        json={"label": label, "millisecondsToExpire": days * 86_400_000},
+        timeout=30,
+        verify=verify,
+    )
+    if not p.ok:
+        raise RuntimeError(f"PAT creation failed [{p.status_code}]: {p.text}")
+    return p.json()
+
+
+
 if __name__ == "__main__":
     if not ENV_PATH.exists():
         raise RuntimeError(f"could not find {ENV_PATH}")
@@ -458,8 +593,59 @@ def run_catalog_bulk_close() -> None:
     #run_deleteview()
 
     #run_createfolder()
-    run_deletefolder()
+    #run_deletefolder()
+
+
+
+
+
+
+
+
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--mode", choices=["azcli", "device", "sp"], default="device")
+    ap.add_argument("--host", default=os.getenv("DREMIO_HOST"))
+    ap.add_argument("--tenant", default=os.getenv("ENTRA_TENANT_ID"))
+    ap.add_argument("--client-id", default=os.getenv("ENTRA_CLIENT_ID"))
+    ap.add_argument("--client-secret", default=os.getenv("ENTRA_CLIENT_SECRET"))
+    ap.add_argument("--scope", default=os.getenv("ENTRA_SCOPE"))
+    ap.add_argument("--create-pat", metavar="USERNAME",
+                    help="also mint a PAT for this Dremio username")
+    ap.add_argument("--pat-label", default="automated")
+    ap.add_argument("--pat-days", type=int, default=90, help="1-180, default 90")
+    ap.add_argument("--ca-bundle", help="path to CA bundle for self-signed Dremio certs")
+    ap.add_argument("--insecure", action="store_true", help="skip TLS verification")
+    args = ap.parse_args()
+
+    missing = [n for n, v in [("--host", args.host), ("--scope", args.scope)] if not v]
+    if args.mode != "azcli":
+        missing += [n for n, v in [("--tenant", args.tenant),
+                                   ("--client-id", args.client_id)] if not v]
+    if args.mode == "sp" and not args.client_secret:
+        missing.append("--client-secret")
+    if missing:
+        ap.error("missing required: " + ", ".join(missing))
+
+    verify: bool | str = args.ca_bundle or (not args.insecure)
+
+    if args.mode == "azcli":
+        jwt = entra_token_azcli(args.scope)
+    elif args.mode == "device":
+        jwt = entra_token_device(args.tenant, args.client_id, args.scope)
+    else:
+        jwt = entra_token_sp(args.tenant, args.client_id, args.client_secret, args.scope)
 
+    tok = dremio_exchange(args.host, jwt, verify=verify)
+    access_token = tok["access_token"]
+    print(f"# Dremio access token (expires in {tok.get('expires_in')}s)", file=sys.stderr)
+    print(access_token)
 
+    if args.create_pat:
+        pat = create_pat(args.host, access_token, args.create_pat,
+                         args.pat_label, args.pat_days, verify=verify)
+        print("# PAT (store it now, it is not retrievable again)", file=sys.stderr)
+        print(pat.get("token") or json.dumps(pat))
 
+    print ("OK")