From efb0edb39f4e2eb4f4207e7b3bb617424c793fde Mon Sep 17 00:00:00 2001 From: nicosammito Date: Sat, 29 Aug 2026 08:39:16 +0100 Subject: [PATCH 1/3] feat: update definition handling to support new JSON structure and improve file processing --- ts/package.json | 2 +- ts/scripts/build-definitions.ts | 55 +++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/ts/package.json b/ts/package.json index d91560d..1fdadc4 100644 --- a/ts/package.json +++ b/ts/package.json @@ -28,7 +28,7 @@ "clean": "rm -rf dist", "typecheck": "tsc --noEmit", "test": "vitest run", - "build:definitions": "npx tsx scripts/build-definitions.ts -- --version def-0.0.35" + "build:definitions": "npx tsx scripts/build-definitions.ts -- --version def-0.0.36" }, "keywords": [], "author": "", diff --git a/ts/scripts/build-definitions.ts b/ts/scripts/build-definitions.ts index b3cd2f7..a389243 100644 --- a/ts/scripts/build-definitions.ts +++ b/ts/scripts/build-definitions.ts @@ -26,10 +26,6 @@ function toPascalCase(str: string): string { .join(""); } -function fileNameToClassName(file: string): string { - return toPascalCase(path.basename(file, ".json")); -} - function toSchemaName(identifier: string): string { // HTTP_METHOD → httpMethodSchema return identifier.toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase()) + "Schema"; @@ -238,21 +234,42 @@ type DefHandler = ( typeFolder: string, ) => void; -function walkDefs(dir: string, relModule: string, handler: DefHandler): void { +// Since def-0.0.36 the release ships one JSON file per module instead of a +// directory tree; every definition is bundled under these list keys. Map each key +// back to the per-type folder name the generators still key off of, so the rest of +// the pipeline (and the generated package layout) stays unchanged. +const LIST_TO_TYPE_FOLDER: Record = { + definitionDataTypes: "data_types", + runtimeFunctionDefinitions: "runtime_functions", + runtimeFlowTypes: "runtime_flow_types", + functionDefinitions: "functions", + flowTypes: "flow_types", +}; +// Runtime functions were filed under their `runtimeName` (`a::b::c` → `a_b_c`); +// everything else under its `identifier`. This reproduces the old on-disk filename. +const RUNTIME_NAME_FOLDERS = new Set(["runtime_functions", "functions"]); + +function defBasename(item: Record, typeFolder: string): string { + return RUNTIME_NAME_FOLDERS.has(typeFolder) + ? String(item.runtimeName ?? "").replace(/::/g, "_") + : String(item.identifier ?? ""); +} + +function walkDefs(dir: string, handler: DefHandler): void { for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - walkDefs(fullPath, path.join(relModule, entry.name), handler); - continue; + if (!entry.isFile() || !entry.name.endsWith(".json")) continue; + const module = JSON.parse(fs.readFileSync(path.join(dir, entry.name), "utf-8")) as Record; + const moduleName = path.basename(entry.name, ".json"); + for (const [listKey, typeFolder] of Object.entries(LIST_TO_TYPE_FOLDER)) { + const items = (module[listKey] as Record[] | undefined) ?? []; + for (const item of items) { + const basename = defBasename(item, typeFolder); + const relModule = path.join(moduleName, typeFolder); + // Upstream data type/flow identifiers are UPPER_SNAKE (e.g. HTTP_METHOD); + // keep our generated tree lowercase to match the rest of the file naming convention. + handler(item, toPascalCase(basename), basename.toLowerCase(), relModule, typeFolder); + } } - if (!entry.name.endsWith(".json") || entry.name === "module.json") continue; - const json = JSON.parse(fs.readFileSync(fullPath, "utf-8")) as Record; - const typeFolder = relModule.split(path.sep).at(-1) ?? ""; - const className = fileNameToClassName(entry.name); - // Upstream data type/flow filenames are UPPER_SNAKE (e.g. HTTP_METHOD.json); - // keep our generated tree lowercase to match the rest of the file naming convention. - const fileName = path.basename(entry.name, ".json").toLowerCase(); - handler(json, className, fileName, relModule, typeFolder); } } @@ -277,7 +294,7 @@ async function main() { // ── Collect all data type definitions ───────────────────────────────────── const dataTypeDefs: DataTypeDef[] = []; - walkDefs(defsDir, "", (json, className, fileName, relModule, typeFolder) => { + walkDefs(defsDir, (json, className, fileName, relModule, typeFolder) => { if (typeFolder !== "data_types" || !json.identifier || !json.type) return; dataTypeDefs.push({ identifier: json.identifier as string, @@ -362,7 +379,7 @@ async function main() { count++; } - walkDefs(defsDir, "", (json, className, fileName, relModule, typeFolder) => { + walkDefs(defsDir, (json, className, fileName, relModule, typeFolder) => { let content: string | null = null; if (typeFolder === "runtime_flow_types") content = generateRuntimeFlowType(json, className, relModule); else if (typeFolder === "runtime_functions") content = generateRuntimeFunction(json, className, relModule); From 6773f78db1f2afd6420e597471538caab9ce4535 Mon Sep 17 00:00:00 2001 From: nicosammito Date: Sat, 29 Aug 2026 08:39:23 +0100 Subject: [PATCH 2/3] feat: update definitions to version def-0.0.36 and adjust JSON handling for module structure --- py/README.md | 4 +-- py/scripts/build_definitions.py | 43 +++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/py/README.md b/py/README.md index 7c8ffc7..b67aa05 100644 --- a/py/README.md +++ b/py/README.md @@ -36,7 +36,7 @@ The built-in data types / functions in `hercules/definitions/` are **generated** release: ```bash -uv run python scripts/build_definitions.py --version def-0.0.35 +uv run python scripts/build_definitions.py --version def-0.0.36 ``` ## Build & release @@ -55,7 +55,7 @@ git tag 0.1.0 && git push origin 0.1.0 ``` The `code0-definition` release used for the built-in definitions is controlled by -the `HERCULES_DEFINITIONS_VERSION` repository variable (default `def-0.0.35`). +the `HERCULES_DEFINITIONS_VERSION` repository variable (default `def-0.0.36`). Build locally: diff --git a/py/scripts/build_definitions.py b/py/scripts/build_definitions.py index 816ead1..654839e 100644 --- a/py/scripts/build_definitions.py +++ b/py/scripts/build_definitions.py @@ -9,7 +9,7 @@ Usage:: - python scripts/build_definitions.py --version def-0.0.35 + python scripts/build_definitions.py --version def-0.0.36 """ from __future__ import annotations @@ -265,16 +265,39 @@ def generate_data_type(dt: "DataTypeDef", ref_map, generic_key_set) -> str: # ── Directory walking ────────────────────────────────────────────────────────── +# Since def-0.0.36 the release ships one JSON file per module instead of a +# directory tree; every definition is bundled under these list keys. Map each key +# back to the per-type folder name the generators still key off of, so the rest of +# the pipeline (and the generated package layout) stays unchanged. +_LIST_TO_TYPE_FOLDER = { + "definitionDataTypes": "data_types", + "runtimeFunctionDefinitions": "runtime_functions", + "runtimeFlowTypes": "runtime_flow_types", + "functionDefinitions": "functions", + "flowTypes": "flow_types", +} +# Runtime functions were filed under their ``runtimeName`` (``a::b::c`` -> ``a_b_c``); +# everything else under its ``identifier``. This reproduces the old on-disk filename. +_RUNTIME_NAME_FOLDERS = {"runtime_functions", "functions"} + + +def _def_basename(item: dict, type_folder: str) -> str: + if type_folder in _RUNTIME_NAME_FOLDERS: + return (item.get("runtimeName") or "").replace("::", "_") + return item.get("identifier") or "" + + def walk_defs(defs_dir: Path): - for path in sorted(defs_dir.rglob("*.json")): - if path.name == "module.json": - continue - rel_module = str(path.parent.relative_to(defs_dir)) - type_folder = path.parent.name - class_name = to_pascal_case(path.stem) - file_name = path.stem.lower() - data = json.loads(path.read_text()) - yield data, class_name, file_name, rel_module, type_folder + for path in sorted(defs_dir.glob("*.json")): + module = json.loads(path.read_text()) + module_name = path.stem + for list_key, type_folder in _LIST_TO_TYPE_FOLDER.items(): + for item in module.get(list_key) or []: + basename = _def_basename(item, type_folder) + rel_module = f"{module_name}/{type_folder}" + class_name = to_pascal_case(basename) + file_name = basename.lower() + yield item, class_name, file_name, rel_module, type_folder def ensure_package(path: Path): From 013067707df7120e2f69b825fee4332343dbce5b Mon Sep 17 00:00:00 2001 From: nicosammito Date: Sat, 29 Aug 2026 08:39:32 +0100 Subject: [PATCH 3/3] feat: update definitions version to def-0.0.36 in build and publish workflows --- .github/workflows/build.yml | 2 +- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5cfff0..f1e60d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: env: # code0-definition release the built-in definitions are generated from. - DEFINITIONS_VERSION: ${{ vars.HERCULES_DEFINITIONS_VERSION || 'def-0.0.35' }} + DEFINITIONS_VERSION: ${{ vars.HERCULES_DEFINITIONS_VERSION || 'def-0.0.36' }} steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 298eb64..29ed177 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,7 +53,7 @@ jobs: env: # code0-definition release the built-in definitions are generated from. - DEFINITIONS_VERSION: ${{ vars.HERCULES_DEFINITIONS_VERSION || 'def-0.0.35' }} + DEFINITIONS_VERSION: ${{ vars.HERCULES_DEFINITIONS_VERSION || 'def-0.0.36' }} steps: - uses: actions/checkout@v6