diff --git a/scripts/gen_fanout_diagram.py b/scripts/gen_fanout_diagram.py new file mode 100644 index 00000000..0c7aa3cc --- /dev/null +++ b/scripts/gen_fanout_diagram.py @@ -0,0 +1,170 @@ +"""Generate the fan-out ingestion figure for ``src/explanation/fan-out-ingestion.md``. + +Why this figure is hand-built when every other diagram in the docs is ``dj.Diagram`` +output +----------------------------------------------------------------------------------- +Because ``dj.Diagram`` cannot draw it. The subject of the page is a ``make()`` that +inserts into tables it holds **no foreign key to**, so there is no dependency for the +renderer to find: point ``dj.Diagram`` at this schema and ``Subject``, ``Session`` and +``Recording`` come out as three unconnected nodes. That absence is exactly what the page +is about, and a figure has to show the write that the dependency graph does not record. + +So the notation is borrowed rather than emitted. Everything a reader already knows how +to read is kept identical to generated diagrams -- see ``src/images/rwm-legend.svg`` and +``how-to/read-diagrams.ipynb``: + +* **Tier by shape and color.** ``Manual`` is a green rounded box, ``Imported`` a blue + ellipse. Fills, strokes and text are the tier colors ``dj.Diagram`` itself emits + (datajoint-python #1544), light and dark. +* **Dependency edges are navy and carry no arrowheads** -- direction follows the layout. +* **Edge weight is cardinality.** ``RecordingFile -> Ingest`` is thick: ``Ingest`` + declares only ``-> RecordingFile``, so the foreign key covers its whole primary key. +* **An underlined name introduces a primary-key attribute of its own.** ``Ingest`` is not + underlined -- it inherits its entire key -- while the three fanned-to tables are. + +The one departure, and it is the point of the figure: **the fan-out writes are drawn +dashed and with arrowheads.** A real dependency edge has no arrowhead, so the arrowhead +is what tells the reader this is not one. The bronze used for renamed foreign keys is +deliberately avoided; these are not foreign keys at all. + +The tables and the ``source_file`` attribute match the code sample on the page, so the +figure and the snippet can be read against each other. + +Usage +----- +No database and no graphviz needed, unlike ``gen_pipeline_diagrams.py``:: + + python scripts/gen_fanout_diagram.py + +Writes ``src/images/fan-out-ingestion.svg``. Idempotent. + +Known limitation, shared with every committed figure here: the dark palette is selected +by ``prefers-color-scheme``, so it follows the reader's operating system rather than the +site's own light/dark toggle. An ````-embedded SVG cannot see the page's +``data-md-color-scheme``. Consistent with the other diagrams; not solved here. +""" + +from pathlib import Path + +OUT = Path(__file__).resolve().parent.parent / "src" / "images" / "fan-out-ingestion.svg" + +W, H = 900, 400 + +# Tier colors as dj.Diagram emits them (fill, stroke, text) -- datajoint-python #1544. +TIER = { + "manual": ("#E8F0E9", "#3E7A52", "#28513A"), + "imported": ("#E0F4FC", "#00A0DF", "#00537A"), +} +TIER_DARK = { + "manual": ("#16281F", "#6BBF94", "#BCE6CF"), + "imported": ("#0F2433", "#33B8E8", "#BEE7F9"), +} +EDGE, EDGE_DARK = "#171C39", "#AEB6C2" # navy, and its dark counterpart +GREY, GREY_DARK = "#808285", "#9DA0A4" # annotations +BG, BG_DARK = "#FFFFFF", "#161A21" + +FONT = "Helvetica, sans-serif" # the face dj.Diagram emits; never monospace + +svg = [] + + +def name(x, y, text, tier, size=15, underline=False): + """A table name. Carries the tier class so the dark block can restyle it -- a CSS + rule outranks a presentation attribute, so the inline fill stays the light default. + """ + svg.append(f'{text}') + if underline: + half = len(text) * size * 0.55 / 2 + svg.append(f'') + + +def manual(cx, cy, label, w=168, h=46, underline=True): + f, s, _ = TIER["manual"] + svg.append(f'') + name(cx, cy + 5, label, "manual", underline=underline) + + +def imported(cx, cy, label, rx=70, ry=26, underline=False): + f, s, _ = TIER["imported"] + svg.append(f'') + name(cx, cy + 5, label, "imported", underline=underline) + + +def main(): + svg.append(f'') + + dark = "\n".join( + f' .f-{k} {{ fill: {v[0]}; stroke: {v[1]}; }}\n' + f' .t-{k} {{ fill: {v[2]}; }}\n' + f' .u-{k} {{ stroke: {v[2]}; }}' for k, v in TIER_DARK.items()) + svg.append(f"""""") + svg.append('' + '') + + CY = 150 + + # the source record, and a genuine dependency: Ingest declares only -> RecordingFile, + # so the foreign key covers its whole primary key -- a thick edge, no arrowhead. + manual(112, CY, "RecordingFile") + svg.append(f'') + imported(336, CY, "Ingest") + + # the fan-out: three entry-point tables with no foreign key back to Ingest + for label, yy in zip(["Subject", "Session", "Recording"], [56, CY, 244]): + manual(752, yy, label) + svg.append(f'') + svg.append(f'insert + source_file') + svg.append(f'no foreign key back to Ingest') + + # legend + ly = 330 + svg.append(f'') + f, s, _ = TIER["imported"] + svg.append(f'') + svg.append(f'Imported') + f, s, _ = TIER["manual"] + svg.append(f'') + svg.append(f'Manual') + + svg.append(f'') + svg.append(f'dependency (no arrowhead)') + svg.append(f'') + svg.append(f'a write, not a dependency') + svg.append(f'' + f'an underlined name introduces a primary-key attribute of its own') + + svg.append("") + OUT.write_text("\n".join(svg) + "\n") + print(f"{OUT.relative_to(Path.cwd())}: written") + + +if __name__ == "__main__": + main() diff --git a/src/explanation/fan-out-ingestion.md b/src/explanation/fan-out-ingestion.md index 77ac883f..b1c0313f 100644 --- a/src/explanation/fan-out-ingestion.md +++ b/src/explanation/fan-out-ingestion.md @@ -14,6 +14,13 @@ loader — reads one source and inserts into several **entry-point tables** (`Manual` or `Imported`) that are *not* foreign-key children of the ingesting table: +![The fan-out ingestion pattern. On the left, RecordingFile, a green rounded box for the Manual tier with its name underlined, joined by a thick navy line without an arrowhead to Ingest, a blue ellipse for the Imported tier whose name is not underlined — the edge is thick because Ingest declares only a foreign key to RecordingFile, so that key covers its whole primary key, and the name is plain because it introduces no key attribute of its own. From Ingest, three dashed navy arrows fan out to the right under the label "insert + source_file", each ending at a green rounded Manual box with an underlined name: Subject, Session and Recording. The arrowheads mark these as writes rather than dependencies, since a dependency edge carries none, and a note reads "no foreign key back to Ingest". A legend keys the Imported and Manual tiers, distinguishes a dependency (no arrowhead) from a write that is not a dependency, and notes that an underlined name introduces a primary-key attribute of its own.](../images/fan-out-ingestion.svg) + +The figure shows what the dependency graph does not: `dj.Diagram` pointed at this +schema renders `Subject`, `Session`, and `Recording` as three unconnected nodes, +because there is no foreign key for it to follow. The dashed arrows are the writes +that the graph cannot record. + ```python @schema class RecordingFile(dj.Manual): # the source record diff --git a/src/images/fan-out-ingestion.svg b/src/images/fan-out-ingestion.svg new file mode 100644 index 00000000..ad5d35c7 --- /dev/null +++ b/src/images/fan-out-ingestion.svg @@ -0,0 +1,51 @@ + + + + +RecordingFile + + + +Ingest + +Subject + + + +Session + + + +Recording + + +insert + source_file +no foreign key back to Ingest + + +Imported + +Manual + +dependency (no arrowhead) + +a write, not a dependency +an underlined name introduces a primary-key attribute of its own +