diff --git a/.github/actions/load-mimic-duckdb/action.yml b/.github/actions/load-mimic-duckdb/action.yml index b8d92f935..e201d1901 100644 --- a/.github/actions/load-mimic-duckdb/action.yml +++ b/.github/actions/load-mimic-duckdb/action.yml @@ -1,5 +1,5 @@ name: "Load MIMIC-IV demo into DuckDB" -description: "Builds a DuckDB database from the demo hosp/icu data using import_duckdb.sh. Requires the duckdb CLI on PATH." +description: "Builds a DuckDB database from the demo hosp/icu data using build_mimic.sh. Requires the duckdb CLI on PATH." inputs: mimic_data_dir: @@ -18,11 +18,17 @@ runs: shell: bash # Resolve paths at shell time so $GITHUB_WORKSPACE points at the # container-mounted path. - # `echo n` answers the overwrite prompt defensively. + # + # Load only. The concepts are transpiled and built by concepts-duckdb.yml + # after this action runs, and the row counts are checked by the calling + # workflow, so build_mimic.sh must not do either here. + env: + MIMIC_MAKE_CONCEPTS: "false" + MIMIC_VALIDATE: "false" run: | data_dir="${{ inputs.mimic_data_dir }}" data_dir="${data_dir:-$GITHUB_WORKSPACE}" db_file="${{ inputs.db_file }}" db_file="${db_file:-$GITHUB_WORKSPACE/mimic4.db}" cd "$GITHUB_WORKSPACE/mimic-iv/buildmimic/duckdb" - echo n | ./import_duckdb.sh "$data_dir" "$db_file" + ./build_mimic.sh "$data_dir" "$db_file" diff --git a/.github/actions/load-mimic-psql/action.yml b/.github/actions/load-mimic-psql/action.yml index 6b91b8ca5..03f9bebae 100644 --- a/.github/actions/load-mimic-psql/action.yml +++ b/.github/actions/load-mimic-psql/action.yml @@ -1,5 +1,5 @@ name: "Load MIMIC-IV demo into PostgreSQL" -description: "Creates the MIMIC-IV schemas and loads the demo hosp/icu data. Requires psql on PATH and PG* connection env vars." +description: "Builds the MIMIC-IV schemas, data, constraints and indexes from the demo hosp/icu data using build_mimic.sh. Requires psql on PATH and PG* connection env vars." inputs: mimic_data_dir: @@ -15,9 +15,11 @@ runs: # Resolve the data dir at shell time so $GITHUB_WORKSPACE points at the # container-mounted path. The github.workspace context yields the host # path, which does not exist inside a container job. + env: + # Load only - we validate/build concepts elsewhere. + MIMIC_MAKE_CONCEPTS: "false" + MIMIC_VALIDATE: "false" run: | data_dir="${{ inputs.mimic_data_dir }}" data_dir="${data_dir:-$GITHUB_WORKSPACE}" - psql -q -v ON_ERROR_STOP=1 -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/create.sql" - psql -q -v ON_ERROR_STOP=1 -v mimic_data_dir="$data_dir" \ - -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/load_gz.sql" + "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/build_mimic.sh" "$data_dir" diff --git a/.github/actions/setup-duckdb/action.yml b/.github/actions/setup-duckdb/action.yml index 91ba9350a..35e3015e5 100644 --- a/.github/actions/setup-duckdb/action.yml +++ b/.github/actions/setup-duckdb/action.yml @@ -5,7 +5,8 @@ inputs: version: description: "DuckDB release version to install." required: false - default: "1.1.3" + # Keep in step with DUCKDB_VERSION in mimic-iv/buildmimic/duckdb/docker. + default: "1.4.5" runs: using: "composite" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..a70e66c2e --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,89 @@ +# Build and run the containerized MIMIC-IV builds against the demo dataset. +name: docker build + +on: + pull_request: + paths: + - 'mimic-iv/buildmimic/postgres/**' + - 'mimic-iv/buildmimic/duckdb/**' + - 'mimic-iv/buildmimic/download_data.sh' + - '.github/workflows/docker-build.yml' + push: + branches: + - main + paths: + - 'mimic-iv/buildmimic/postgres/**' + - 'mimic-iv/buildmimic/duckdb/**' + - '.github/workflows/docker-build.yml' + +# Cancel superseded runs when a PR is pushed again. +concurrency: + group: docker-build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + defaults: + run: + # Name bash explicitly to get `-eo pipefail`. + shell: bash + + env: + DUCKDB_DOCKER: mimic-iv/buildmimic/duckdb/docker + POSTGRES_DOCKER: mimic-iv/buildmimic/postgres/docker + MIMIC_DATA_DIR: ${{ github.workspace }} + MIMIC_OUTPUT_DIR: ${{ github.workspace }}/duckdb-out + + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/download-demo + - uses: ./.github/actions/setup-duckdb + + - name: Build both images + run: | + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" build + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" build + + # --exit-code-from implies --abort-on-container-exit and returns the build + # job's own status, so a failed build fails the step. + - name: Build MIMIC-IV in DuckDB + run: | + mkdir -p "$MIMIC_OUTPUT_DIR" + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" \ + up --exit-code-from mimic-build | tee duckdb.log + if grep -F -q "FAILED" duckdb.log; then + echo "::error::DuckDB container row-count validation failed:" + grep -F "FAILED" duckdb.log + exit 1 + fi + + - name: Read the built database with the runner's own duckdb + run: | + rows=$(duckdb "$MIMIC_OUTPUT_DIR/mimic4.db" -noheader -list \ + -c "SELECT count(*) FROM mimiciv_derived.sepsis3") + echo "mimiciv_derived.sepsis3: ${rows} rows" + if [ -z "${rows}" ] || [ "${rows}" -eq 0 ]; then + echo "::error::The concepts are missing from the container-built database." + exit 1 + fi + + - name: Build MIMIC-IV in PostgreSQL + run: | + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" \ + up --exit-code-from mimic-build | tee psql.log + if grep -F -q "FAILED" psql.log; then + echo "::error::PostgreSQL container row-count validation failed:" + grep -F "FAILED" psql.log + exit 1 + fi + + - name: Tear down + if: always() + run: | + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" down -v || true + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" down -v || true diff --git a/.gitignore b/.gitignore index d0f33f313..661750085 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -## Local docker-postgres raw data (do not commit large CSVs) -mimic-iv/docker-postgres/mimic-data/ +## Raw data downloaded for local builds (do not commit large CSVs) +mimic-data/ ## Allow example env, ignore real env -mimic-iv/docker-postgres/.env +.env +!.env.example # duckdb / sqlite db files *.db diff --git a/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json b/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json new file mode 100644 index 000000000..a19efcd57 --- /dev/null +++ b/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json @@ -0,0 +1,7 @@ +[ + { + "name": "subject_id", + "type": "INT64", + "mode": "REQUIRED" + } +] \ No newline at end of file diff --git a/mimic-iv/buildmimic/download_data.sh b/mimic-iv/buildmimic/download_data.sh new file mode 100755 index 000000000..0131a6999 --- /dev/null +++ b/mimic-iv/buildmimic/download_data.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Download MIMIC-IV from PhysioNet into a directory any of the builds can use. +# +# Usage: ./download_data.sh [destination] [physionet-username] +# +# destination where to put the data (default ./mimic-data) +# physionet-username also read from $PHYSIONET_USER, otherwise prompted for +# +# Requires a PhysioNet account credentialed for MIMIC-IV. The password is always +# requested interactively, so it never reaches the process list or shell history. +# +# Shared by the postgres and duckdb builds: both want the same hosp/ and icu/ +# layout, so there is one copy of this here rather than one per engine. +# +# This is a convenience only. If you already have the data, point MIMIC_DATA_DIR +# at it instead; any directory with hosp/ and icu/ subfolders will do. +set -euo pipefail + +# The build scripts target the current release of MIMIC-IV. +readonly MIMIC_VERSION="3.1" + +DEST="${1:-./mimic-data}" +USERNAME="${2:-${PHYSIONET_USER:-}}" + +if [ -z "${USERNAME}" ]; then + read -rp "PhysioNet username: " USERNAME +fi + +echo "Downloading MIMIC-IV v${MIMIC_VERSION} to ${DEST}" + +# -nH --cut-dirs=4 strips physionet.org/files/mimiciv// from the paths, +# leaving the hosp/ and icu/ subfolders directly under ${DEST}. That is the +# layout buildmimic/postgres/load_gz.sql expects. +wget -r -N -c -np -nH --cut-dirs=4 \ + -A '*.csv.gz' \ + -P "${DEST}" \ + --user "${USERNAME}" --ask-password \ + "https://physionet.org/files/mimiciv/${MIMIC_VERSION}/" + +echo "Downloaded $(find "${DEST}" -name '*.csv.gz' | wc -l | tr -d ' ') files to ${DEST}" diff --git a/mimic-iv/buildmimic/duckdb/README.md b/mimic-iv/buildmimic/duckdb/README.md index 60bbd3d52..28a4f728b 100644 --- a/mimic-iv/buildmimic/duckdb/README.md +++ b/mimic-iv/buildmimic/duckdb/README.md @@ -24,12 +24,15 @@ which you can obtain by either installing [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) or [Cygwin](https://www.cygwin.com/). +If you would rather not install DuckDB at all, the [docker](docker) folder +builds the same database in a container with a pinned DuckDB version. + ## Set-up ### Quick overview 1. [Install](https://duckdb.org/docs/installation/) the CLI version of DuckDB -2. [Download](https://physionet.org/content/mimiciv/2.0) the MIMIC-IV files +2. [Download](https://physionet.org/content/mimiciv/) the MIMIC-IV files 3. Create DuckDB database and load data ### Install DuckDB @@ -41,6 +44,9 @@ the CLI version of DuckDB. You will need to place the `duckdb` binary in a folder on your environment path, e.g. `/usr/local/bin`. +These scripts are built and tested against the 1.4.x LTS line (currently +1.4.5), which is what CI uses. + ### Download MIMIC-IV files Download the CSV files for [MIMIC-IV](https://physionet.org/content/mimiciv/) @@ -63,43 +69,38 @@ mimic_data_dir The CSV files can be uncompressed (end in `.csv`) or compressed (end in `.csv.gz`). -The easiest way to download them is to open a terminal then run: +The easiest way to download them is to use the shared download script, which +wraps `wget` and puts the files in the layout the build expects: -``` -wget -r -N -c -np --user YOURUSERNAME --ask-password https://physionet.org/files/mimiciv/2.2/ +```sh +../download_data.sh ./mimic-data YOURUSERNAME ``` -Replace `YOURUSERNAME` with your physionet username. - -This will make you `mimic_data_dir` be `physionet.org/files/mimiciv/2.2`. +Replace `YOURUSERNAME` with your physionet username; you will be prompted for +the password. This makes your `mimic_data_dir` be `./mimic-data`. # Create DuckDB database and load data The last step requires creating a DuckDB database and loading the data into it. -You can do all of this with one shell script, `import_duckdb.sh`, +You can do all of this with one shell script, `build_mimic.sh`, located in this repository. -See the help for it below: - ```sh -$ ./import_duckdb.sh -h -./import_duckdb.sh: -USAGE: ./import_duckdb.sh mimic_data_dir [output_db] -WHERE: - mimic_data_dir directory that contains csv.gz or csv files - output_db: optional filename for duckdb file (default: mimic4.db) -$ +$ ./build_mimic.sh -h +Usage: build_mimic.sh [output_db] + mimic_data_dir the directory containing the hosp/ and icu/ subfolders + output_db filename for the duckdb file (default mimic4.db) ``` Here's an example invocation that will make the database in the default "mimic4.db": ```sh -$ ./import_duckdb.sh physionet.org/files/mimiciv/2.2 +$ ./build_mimic.sh ./mimic-data <... output of script snipped ...> -Successfully finished loading data into mimic4.db. +MIMIC-IV build complete: /path/to/mimic4.db $ ls -lh mimic4.db -rw-rw-r--. 1 myuser mygroup 93G May 26 16:11 mimic4.db @@ -109,6 +110,15 @@ The script will print out progress as it goes. Be patient, this can take minutes to hours to load depending on your computer's configuration. +Beyond loading the data, it also derives the concepts from +[concepts_duckdb](../../concepts_duckdb) into the `mimiciv_derived` schema and +checks the loaded tables against known row counts. Set `MIMIC_MAKE_CONCEPTS` or +`MIMIC_VALIDATE` to `false` to skip either. + +Each step is recorded in a `mimiciv_build_progress` table inside the database +file, and tables that already hold rows are skipped, so re-running the script +after an interruption resumes rather than starting over. + * It took 16m25s on a Fedora 34 workstation with duckdb v 0.2.6, a btrfs filesystem with ztsd level 1 compression, AMD Ryzen 3900X, 32 GB RAM, Samsung 970 Evo NVMe SSD. * It took ~10m on a Mac M1 Max 2021, 32 GB RAM. diff --git a/mimic-iv/buildmimic/duckdb/build_mimic.sh b/mimic-iv/buildmimic/duckdb/build_mimic.sh new file mode 100755 index 000000000..048819988 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/build_mimic.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Build MIMIC-IV in DuckDB: create the schema, load the data, then derive the +# concepts. +# +# Usage: +# ./build_mimic.sh [output_db] +# +# is the directory holding the hosp/ and icu/ subfolders. +# Compressed (.csv.gz) and uncompressed (.csv) data are both supported; DuckDB +# decompresses by file extension, so no separate loader is needed. +# +# Each step is recorded in a progress table once it completes, so an +# interrupted build will resume where it stopped. The table lives inside the +# database file, so the resume state travels with it. +# +# Environment: +# MIMIC_DATA_DIR used when is not given +# MIMIC_DB used when [output_db] is not given (default mimic4.db) +# MIMIC_MAKE_CONCEPTS derive the concepts into mimiciv_derived (default true) +# MIMIC_VALIDATE check tables against expected row counts (default true) +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +readonly SCRIPT_DIR +readonly CONCEPTS_DIR="${SCRIPT_DIR}/../../concepts_duckdb" +# The schema and the row count checks are shared with the postgres build rather +# than duplicated. create.sql needs the patches in patch_create_sql below; +# validate.sql is plain ANSI SQL and runs unmodified. +readonly POSTGRES_DIR="${SCRIPT_DIR}/../postgres" +readonly PROGRESS_TABLE=mimiciv_build_progress + +DATA_DIR="" +DB="" + +# -f tells DuckDB to stop at the first failing statement and exit non-zero. +duckdb_run() { duckdb "${DB}" -f "$1"; } +duckdb_cmd() { duckdb "${DB}" -c "$1"; } +duckdb_val() { duckdb "${DB}" -noheader -list -c "$1"; } + +# Lowercase string before checking value +is_true() { [ "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" = "true" ]; } + +usage() { + echo "Usage: $(basename "$0") [output_db]" >&2 + echo " mimic_data_dir the directory containing the hosp/ and icu/ subfolders" >&2 + echo " output_db filename for the duckdb file (default mimic4.db)" >&2 +} + +step_done() { + [ "$(duckdb_val "SELECT EXISTS (SELECT 1 FROM ${PROGRESS_TABLE} WHERE step = '$1')")" = "true" ] +} + +run_step() { + local step=$1 + shift + if step_done "${step}"; then + echo "== ${step}: already done, skipping" + return + fi + echo "== ${step}: starting" + "$@" + duckdb_cmd "INSERT INTO ${PROGRESS_TABLE} (step) VALUES ('${step}') ON CONFLICT DO NOTHING" >/dev/null + echo "== ${step}: done" +} + +# The schema is defined once, in the postgres build. Three things in it are not +# valid or not desirable in DuckDB: +# 1. TIMESTAMP(NN) -- DuckDB does not accept a precision argument. +# 2. NOT NULL on mimiciv_hosp.microbiologyevents.spec_type_desc -- there is one +# (!) zero-length string, which the import treats as NULL. +# 3. NOT NULL on mimiciv_hosp.prescriptions.drug -- likewise, zero-length +# strings treated as NULL. +patch_create_sql() { + sed -E \ + -e 's/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' \ + -e 's/spec_type_desc(.+)NOT NULL/spec_type_desc\1/g' \ + -e 's/drug +(VARCHAR.+)NOT NULL/drug \1/g' \ + "${POSTGRES_DIR}/create.sql" +} + +create_schema() { + patch_create_sql | duckdb "${DB}" +} + +# hosp/admissions.csv.gz -> mimiciv_hosp.admissions +make_table_name() { + local path=$1 basename dirname + basename=${path##*/} + dirname=${path%/*} + dirname=${dirname##*/} + printf 'mimiciv_%s.%s' "${dirname}" "${basename%%.*}" +} + +# A COPY is a single transaction, so a table is either fully loaded or empty. +# Even a single row indicates completion of the data load. +table_has_rows() { + [ "$(duckdb_val "SELECT EXISTS (SELECT 1 FROM $1 LIMIT 1)")" = "true" ] +} + +load_data() { + local file table + while IFS= read -r file; do + table=$(make_table_name "${file}") + + # A file with no matching table in create.sql is reported and skipped + # rather than failing the whole build. + if ! duckdb_val "SELECT 1 FROM ${table} LIMIT 0" >/dev/null 2>&1; then + echo " ${table}: not in the schema, skipping ${file##*/}" + continue + fi + if table_has_rows "${table}"; then + echo " ${table}: already loaded, skipping" + continue + fi + echo " ${table}: loading" + duckdb_cmd "COPY ${table} FROM '${file}' (HEADER, DELIM ',', QUOTE '\"', ESCAPE '\"')" >/dev/null + done < <(find "${DATA_DIR}"/hosp "${DATA_DIR}"/icu -type f \ + \( -name '*.csv' -o -name '*.csv.gz' \) | sort) +} + +make_concepts() { + # duckdb.sql pulls in the individual concepts with .read, which resolves + # relative to the working directory rather than to the script. + cd "${CONCEPTS_DIR}" + duckdb "${DB}" -f duckdb.sql +} + +validate() { + # default validate the full DB + local script=validate.sql + + # The demo is a 100 patient subset of MIMIC-IV, so it has its own set of + # expected row counts. + if [ "$(duckdb_val 'SELECT count(*) FROM mimiciv_hosp.patients')" -eq 100 ]; then + echo " 100 patients found, validating against the MIMIC-IV demo counts" + script=validate_demo.sql + fi + + local output + output=$(duckdb "${DB}" -f "${POSTGRES_DIR}/${script}") + echo "${output}" + + # Mismatches indicate (1) failure to load (zero rows) or (2) mismatch mimic version + if echo "${output}" | grep -q FAILED; then + echo "WARNING: some tables do not have the expected number of rows." >&2 + fi +} + +resolve_data_dir() { + local dir=${1:-${MIMIC_DATA_DIR:-}} + + if [ -z "${dir}" ]; then + usage + exit 1 + fi + if [ ! -d "${dir}/hosp" ] || [ ! -d "${dir}/icu" ]; then + echo "ERROR: ${dir} must contain the hosp/ and icu/ subfolders of MIMIC-IV." >&2 + exit 1 + fi + if [ -z "$(find "${dir}/hosp" -maxdepth 1 \( -name '*.csv' -o -name '*.csv.gz' \) -print -quit)" ]; then + echo "ERROR: no .csv or .csv.gz files found in ${dir}/hosp." >&2 + exit 1 + fi + + DATA_DIR=$(cd -- "${dir}" && pwd) +} + +# make_concepts has to cd into the concepts directory, so a relative path here +# would resolve against the wrong directory and silently build a second, +# empty database. +resolve_db() { + local db=${1:-${MIMIC_DB:-mimic4.db}} + local dir=${db%/*} + + [ "${dir}" = "${db}" ] && dir=. + if [ ! -d "${dir}" ]; then + echo "ERROR: ${dir} does not exist, cannot create ${db} in it." >&2 + exit 1 + fi + + DB="$(cd -- "${dir}" && pwd)/${db##*/}" +} + +main() { + case "${1:-}" in + -h|--help) usage; exit 0 ;; + esac + + resolve_data_dir "${1:-}" + resolve_db "${2:-}" + echo "Loading from ${DATA_DIR} into ${DB} using DuckDB $(duckdb --version)" + + duckdb_cmd "CREATE TABLE IF NOT EXISTS ${PROGRESS_TABLE} ( + step text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() + )" >/dev/null + + run_step create create_schema + run_step load load_data + + if is_true "${MIMIC_MAKE_CONCEPTS:-true}"; then + run_step concepts make_concepts + else + echo "== concepts: MIMIC_MAKE_CONCEPTS is not 'true', skipping" + fi + + if is_true "${MIMIC_VALIDATE:-true}"; then + run_step validate validate + else + echo "== validate: MIMIC_VALIDATE is not 'true', skipping" + fi + + echo "MIMIC-IV build complete: ${DB}" +} + +main "$@" diff --git a/mimic-iv/buildmimic/duckdb/docker/.env.example b/mimic-iv/buildmimic/duckdb/docker/.env.example new file mode 100644 index 000000000..f2b7a9b0d --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/.env.example @@ -0,0 +1,20 @@ +# Copy to .env and edit as needed: cp .env.example .env + +# DuckDB version used to build the database. 1.4.x is the current LTS. +DUCKDB_VERSION=1.4.5 + +# Directory holding the MIMIC-IV .csv.gz files, containing hosp/ and icu/ +# subfolders. Populate it by running ../../download_data.sh +MIMIC_DATA_DIR=./mimic-data + +# Where the finished database is written on the host, and what to call it. +MIMIC_OUTPUT_DIR=./mimic-db +MIMIC_DB_NAME=mimic4.db + +# Derive the concepts from mimic-iv/concepts_duckdb once the data is loaded. +# Set to false to load the raw tables only. +MIMIC_MAKE_CONCEPTS=true + +# Check the loaded tables against the expected row counts. The demo and the full +# dataset are detected automatically. +MIMIC_VALIDATE=true diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile b/mimic-iv/buildmimic/duckdb/docker/Dockerfile new file mode 100644 index 000000000..95bfc9c9c --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile @@ -0,0 +1,41 @@ +# Image for the job that builds MIMIC-IV into a DuckDB file. +# Writes a .db file into a mounted directory and exits. +# +# The build context is the mimic-iv/ directory, and the repository layout is +# kept intact inside the image so that build_mimic.sh finds the shared postgres +# schema beside it and the concepts two directories up. +# +# There is no official DuckDB image, so the CLI is fetched from the pinned +# GitHub release. DUCKDB_VERSION decides which one; see docker-compose.yml. +ARG DUCKDB_VERSION=1.4.5 + +FROM debian:stable-slim + +# Repeated because the ARG above the first FROM is outside the build stage. +ARG DUCKDB_VERSION +# Set automatically by buildkit: amd64 or arm64. DuckDB publishes both under +# exactly these names. +ARG TARGETARCH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl unzip \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL -o /tmp/duckdb.zip \ + "https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-${TARGETARCH}.zip" \ + && unzip -q /tmp/duckdb.zip -d /usr/local/bin \ + && rm /tmp/duckdb.zip \ + && chmod +x /usr/local/bin/duckdb \ + && duckdb --version + +COPY buildmimic/duckdb/build_mimic.sh /mimic/buildmimic/duckdb/ +COPY buildmimic/duckdb/docker/docker-entrypoint.sh /mimic/ +# create.sql, validate.sql and validate_demo.sql are shared with the postgres +# build; build_mimic.sh reads them from ../postgres. +COPY buildmimic/postgres/create.sql buildmimic/postgres/validate*.sql /mimic/buildmimic/postgres/ +COPY concepts_duckdb/ /mimic/concepts_duckdb/ + +# The database is written here. +WORKDIR /out + +ENTRYPOINT ["/mimic/docker-entrypoint.sh"] +CMD ["/data"] diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..701d3b5cc --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore @@ -0,0 +1,11 @@ +# The build context is mimic-iv/ +# Exclude everything, then add back only what the image needs. +* +!buildmimic/duckdb/build_mimic.sh +!buildmimic/duckdb/docker/docker-entrypoint.sh +!buildmimic/postgres/create.sql +!buildmimic/postgres/validate.sql +!buildmimic/postgres/validate_demo.sql +!concepts_duckdb/ +# Explicitly avoid including output db files +*.db diff --git a/mimic-iv/buildmimic/duckdb/docker/README.md b/mimic-iv/buildmimic/duckdb/docker/README.md new file mode 100644 index 000000000..527ed74ff --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/README.md @@ -0,0 +1,103 @@ +# MIMIC-IV with DuckDB and Docker + +Build a MIMIC-IV DuckDB database in a container. The build scripts live in the +parent folder. It creates the schema, loads the data, (optionally, default true) +derives the concepts from [concepts_duckdb](../../../concepts_duckdb), and +validates the result. The container is only used to reproducibly build the +database: it is not necessary for analyzing it afterward. + +## Requirements + +* Docker, with Compose v2. +* The dataset (`../../download_data.sh` is a convenience for downloading data from PhysioNet). +* Disk space. The compressed download is about 10 GB; the resulting `.db` file + is roughly 25 GB, plus a few GB if you build the concepts. The + database is written to a bind mount on your host, so this space comes from + your disk rather than the Docker Desktop virtual disk, but the *load* still + needs Docker to have room to work. + +## Quickstart + +```bash +# 1. Configure. Set DUCKDB_VERSION to match your local duckdb. +# Make sure MIMIC_DATA_DIR has the hosp/ and icu/ subfolders. +cp .env.example .env + +# 2. Download the data, if you do not already have it (about 10 GB). +../../download_data.sh + +# 3. Build. Runs in the foreground so you can watch it; add -d to detach. +docker compose up +``` + +## Notes + +### DuckDB version + +The container's DuckDB and the `duckdb` on your machine are two separate installs. +The container has 1.4.5 which is the LTS version. `duckdb` is mostly backward +compatible but be aware of differences. + +Check what you have: + +```bash +duckdb --version +``` + +and set `DUCKDB_VERSION` in `.env` to match before building. If you later see an +error about the storage version when opening the file, the fix is to upgrade +your local DuckDB or rebuild with a lower `DUCKDB_VERSION`. + +The result is `./mimic-db/mimic4.db`. Change `MIMIC_OUTPUT_DIR` and +`MIMIC_DB_NAME` in `.env` to put it somewhere else. + +Loading takes on the order of an hour for the full dataset on a reasonable +machine, plus time for the concepts. Watch it with: + +```bash +docker compose logs -f mimic-build +``` + +### If the build is interrupted + +Run `docker compose up` again. The build records each step as it completes and +skips tables that are already populated, so it resumes rather than starting +over. Progress is tracked in the `mimiciv_build_progress` table inside the +database file. + +```bash +duckdb ./mimic-db/mimic4.db -c 'TABLE mimiciv_build_progress' +``` + +To discard everything and start clean, delete the file: + +```bash +rm ./mimic-db/mimic4.db +docker compose up --build +``` + +Use `--build` whenever you change the SQL or `DUCKDB_VERSION`, otherwise +Compose reuses the existing image. + +### Settings + +All are set in `.env`; see `.env.example` for the defaults. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `DUCKDB_VERSION` | `1.4.5` | DuckDB version to build with. Match your local `duckdb`. | +| `MIMIC_DATA_DIR` | `./mimic-data` | Directory holding `hosp/` and `icu/`. | +| `MIMIC_OUTPUT_DIR` | `./mimic-db` | Host directory the database is written to. | +| `MIMIC_DB_NAME` | `mimic4.db` | Filename of the database. | +| `MIMIC_MAKE_CONCEPTS` | `true` | Derive the concepts into `mimiciv_derived`. | +| `MIMIC_VALIDATE` | `true` | Check the loaded tables against expected row counts. | + +## Using the database + +```bash +# from the host, with your own duckdb +duckdb ./mimic-db/mimic4.db + +# otherwise, from inside a throwaway container using the pinned version (not recommended) +docker compose run --rm --entrypoint duckdb mimic-build /out/mimic4.db +``` diff --git a/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml new file mode 100644 index 000000000..2bf9d9990 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml @@ -0,0 +1,24 @@ +name: mimic-duckdb + +services: + # Loads the data and derives the concepts, then exits. + # Re-running it resumes an interrupted build. + # + # As DuckDB is embedded, this container's only purpose is to + # reproducibly create the database file. + mimic-build: + build: + # mimic-iv/, so the build can reach buildmimic/ and concepts_duckdb + context: ../../.. + dockerfile: buildmimic/duckdb/docker/Dockerfile + args: + DUCKDB_VERSION: ${DUCKDB_VERSION:-1.4.5} + environment: + MIMIC_DB: /out/${MIMIC_DB_NAME:-mimic4.db} + MIMIC_MAKE_CONCEPTS: ${MIMIC_MAKE_CONCEPTS:-true} + MIMIC_VALIDATE: ${MIMIC_VALIDATE:-true} + volumes: + # The .csv.gz files, read-only. Must contain hosp/ and icu/ subfolders. + - ${MIMIC_DATA_DIR:-./mimic-data}:/data:ro + # The finished database is output to a bind mount. + - ${MIMIC_OUTPUT_DIR:-./mimic-db}:/out diff --git a/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh b/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh new file mode 100755 index 000000000..ee96c0ba7 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# This entrypoint exists as we need to infer the user ID from the bind mount to +# re-assign ownership of the output db file from root to the user. +set -eu + +give_output_to_host_user() { + chown -R --reference=/out /out 2>/dev/null || true +} +trap give_output_to_host_user EXIT + +/mimic/buildmimic/duckdb/build_mimic.sh "$@" diff --git a/mimic-iv/buildmimic/duckdb/filter_db.sql b/mimic-iv/buildmimic/duckdb/filter_db.sql new file mode 100644 index 000000000..c152c366f --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/filter_db.sql @@ -0,0 +1,45 @@ +-- remove all but the first 10 subjects +-- mimiciv_hosp.demo_subject_id +DELETE FROM mimiciv_hosp.patients +WHERE subject_id NOT IN +(SELECT subject_id FROM mimiciv_hosp.patients ORDER BY subject_id LIMIT 10); + +-- apply this to all the other tables +DELETE FROM mimiciv_hosp.admissions WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.diagnoses_icd WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.drgcodes WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.emar WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.emar_detail WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.hcpcsevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.labevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.microbiologyevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.omr WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.pharmacy WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.poe WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.poe_detail WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.prescriptions WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.procedures_icd WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.services WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.transfers WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); + +DELETE FROM mimiciv_icu.chartevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.datetimeevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.icustays WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.ingredientevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.inputevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.outputevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.procedureevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); + + +-- d_hcpcs +-- d_icd_diagnoses +-- d_icd_procedures +-- d_items +-- d_labitems +-- caregiver +-- provider + +-- to reduce the filesize, the database will need to be exported and reimported +EXPORT DATABASE 'tmp_output'; + +IMPORT DATABASE 'tmp_output'; \ No newline at end of file diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh deleted file mode 100755 index 4ff9fc219..000000000 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2023 MIT Laboratory for Computational Physiology -# Copyright (c) 2021 Thomas Ward -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -yell () { echo "$0: $*" >&2; } -die () { yell "$*"; exit 111; } -try () { "$@" || die "Exiting. Failed to run: \"$*\""; } - -usage () { - die " -USAGE: ./import_duckdb.sh mimic_data_dir [output_db] -WHERE: - mimic_data_dir directory that contains csv.gz or csv files - output_db: optional filename for duckdb file (default: mimic4.db)\ -" -} - -# Print help if requested -echo "$0 $* " | grep -Eq " -h | --help " && usage - -# rename CLI positional args to more friendly variable names -MIMIC_DIR=$1 -# allow optional specification of duckdb name, otherwise default to mimic4.db -OUTFILE=mimic4.db -if [ -n "$2" ]; then - OUTFILE=$2 -fi - - -# basic error checking before running -if [ -z "$MIMIC_DIR" ]; then - yell "Please specify a mimic data directory" - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ ! -d "$MIMIC_DIR" ]; then - yell "Specified directory \"$MIMIC_DIR\" does not exist." - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ -n "$3" ]; then - yell "import_duckdb.sh takes a maximum of two arguments." - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ -s "$OUTFILE" ]; then - yell "File \"$OUTFILE\" already exists." - printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " - read -r yn - case $yn in - [Yy]* ) ;; # OK - [Nn]* ) exit;; - [Dd]* ) rm "$OUTFILE";; - * ) die "Unrecognized input.";; - esac -fi - -# we will copy the postgresql create.sql file, and apply regex -# to fix the following issues: -# 1. Remove optional precision value from TIMESTAMP(NN) -> TIMESTAMP -# duckdb does not support this. -export REGEX_TIMESTAMP='s/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' -# 2. Remove NOT NULL constraint from mimiciv_hosp.microbiologyevents.spec_type_desc -# as there is one (!) zero-length string which is treated as a NULL by the import. -export REGEX_SPEC_TYPE='s/spec_type_desc(.+)NOT NULL/spec_type_desc\1/g' -# 3. Remove NOT NULL constraint from mimiciv_hosp.prescriptions.drug -# as there are zero-length strings which are treated as NULLs by the import. -export REGEX_DRUG='s/drug +(VARCHAR.+)NOT NULL/drug \1/g' - -# use sed + above regex to create tables within db -sed -E -e "${REGEX_TIMESTAMP}" ../postgres/create.sql | \ - sed -E -e "${REGEX_SPEC_TYPE}" | \ - sed -E -e "${REGEX_DRUG}" | \ - duckdb "$OUTFILE" - -# goal: get path from find, e.g., ./1.0/icu/d_items -# and return database table name for it, e.g., mimiciv_icu.d_items -make_table_name () { - # strip leading directories (e.g., ./icu/hello.csv.gz -> hello.csv.gz) - BASENAME=${1##*/} - # strip suffix (e.g., hello.csv.gz -> hello; hello.csv -> hello) - TABLE_NAME=${BASENAME%%.*} - # strip basename (e.g., ./icu/hello.csv.gz -> ./icu) - PATHNAME=${1%/*} - # strip leading directories from PATHNAME (e.g. ./icu -> icu) - DIRNAME=${PATHNAME##*/} - TABLE_NAME="mimiciv_$DIRNAME.$TABLE_NAME" -} - - -# load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do - make_table_name "$FILE" - - # skip directories which we do not expect in mimic-iv - # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in - (hosp|icu) ;; # OK - (*) continue; - esac - echo "Loading $FILE .. \c" - OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); -EOSQL - ) - # If the table is missing in the DB, we emit a warning and continue. - # Otherwise, the script repeats the error and exits. - STATUS=$? - if [ $STATUS -ne 0 ]; then - echo "$OUTPUT" | grep -qiE 'table .* does not exist' && { - echo "skipped (table $TABLE_NAME not found)"; - continue; - } - yell "Failed loading $FILE into $TABLE_NAME" - yell "$OUTPUT" - die "Exiting due to load error." - fi - echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/postgres/README.md b/mimic-iv/buildmimic/postgres/README.md index 5ab28c2c4..6b729450f 100644 --- a/mimic-iv/buildmimic/postgres/README.md +++ b/mimic-iv/buildmimic/postgres/README.md @@ -20,6 +20,37 @@ cd mimic-code wget -r -N -c -np --user --ask-password https://physionet.org/files/mimiciv/3.1/ mv physionet.org/files/mimiciv mimiciv && rmdir physionet.org/files && rm physionet.org/robots.txt && rmdir physionet.org createdb mimiciv +PGDATABASE=mimiciv mimic-iv/buildmimic/postgres/build_mimic.sh mimiciv/3.1 +``` + +`build_mimic.sh` creates the schema, loads the data, adds constraints/indexes, +and optionally (default true) derives the [concepts](../../concepts_postgres). + +Compressed (`.csv.gz`) and uncompressed (`.csv`) data are both +detected, and the connection is taken from the standard `PG*` environment +variables. + +It is safe to re-run. Each step is recorded once it completes, and the load +skips tables that already hold rows, so an interrupted build resumes rather than +starting over. This matters most for `chartevents`, which takes by far the +longest to load. Progress is kept in the `mimiciv_build_progress` table. + +Two steps can be turned off: + +```sh +MIMIC_MAKE_CONCEPTS=false MIMIC_VALIDATE=false PGDATABASE=mimiciv \ + mimic-iv/buildmimic/postgres/build_mimic.sh mimiciv/3.1 +``` + +To build in a container instead, see [docker](docker/), which runs this same +script against a containerized PostgreSQL. + +## Running the steps individually + +The build script is a wrapper around the SQL files in this directory, which can +equally be run by hand: + +```sh psql -d mimiciv -f mimic-iv/buildmimic/postgres/create.sql psql -d mimiciv -v ON_ERROR_STOP=1 -v mimic_data_dir=mimiciv/3.1 -f mimic-iv/buildmimic/postgres/load_gz.sql psql -d mimiciv -v ON_ERROR_STOP=1 -v mimic_data_dir=mimiciv/3.1 -f mimic-iv/buildmimic/postgres/constraint.sql diff --git a/mimic-iv/buildmimic/postgres/build_mimic.sh b/mimic-iv/buildmimic/postgres/build_mimic.sh new file mode 100755 index 000000000..fd96987f7 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/build_mimic.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Build MIMIC-IV in PostgreSQL: create the schema, load the data, add the +# constraints and indexes, then derive the concepts. +# +# Usage: +# ./build_mimic.sh +# +# is the directory holding the hosp/ and icu/ subfolders. +# Compressed (.csv.gz) and uncompressed (.csv) data are both supported. +# +# Each step is recorded in a progress table once it completes, so an +# interrupted build will resume where it stopped. +# +# Environment: +# MIMIC_DATA_DIR used when is not given +# MIMIC_MAKE_CONCEPTS derive the concepts into mimiciv_derived (default true) +# MIMIC_VALIDATE check tables against expected row counts (default true) +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +readonly SCRIPT_DIR +readonly CONCEPTS_DIR="${SCRIPT_DIR}/../../concepts_postgres" +readonly PROGRESS_TABLE=public.mimiciv_build_progress + +# load.sql and load_gz.sql set this for their session; the load is run statement +# by statement here, so set it for every connection instead. +export PGCLIENTENCODING=UTF8 + +DATA_DIR="" +LOADER="" + +psql_run() { psql -v ON_ERROR_STOP=1 --quiet "$@"; } +psql_val() { psql -v ON_ERROR_STOP=1 -Atq -c "$1"; } + +# Lowercase string before checking value +is_true() { [ "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" = "true" ]; } + +usage() { + echo "Usage: $(basename "$0") " >&2 + echo " the directory containing the hosp/ and icu/ subfolders" >&2 +} + +# Completed steps are recorded in a table located on the public schema. +step_done() { + [ "$(psql_val "SELECT EXISTS (SELECT 1 FROM ${PROGRESS_TABLE} WHERE step = '$1')")" = "t" ] +} + +run_step() { + local step=$1 + shift + if step_done "${step}"; then + echo "== ${step}: already done, skipping" + return + fi + echo "== ${step}: starting" + "$@" + psql_run -c "INSERT INTO ${PROGRESS_TABLE} (step) VALUES ('${step}') ON CONFLICT DO NOTHING" + echo "== ${step}: done" +} + +create_schema() { + psql_run -f "${SCRIPT_DIR}/create.sql" +} + +# To reduce redundancy, this script reads the \COPY statements to determine +# the tables to build, and builds them in order. +parse_copy_statements() { + awk -F' ' ' + /^\\cd / { + dir = $2 + if (dir ~ /^:/) { cur = "" } # \cd :mimic_data_dir + else { sub(/^\.\.\//, "", dir); cur = dir } # \cd hosp, \cd ../icu + next + } + /^\\COPY / { print cur "\t" $2 "\t" $0 } + ' "${SCRIPT_DIR}/${LOADER}" +} + +# A \COPY is a single statement, so a table is either fully loaded or empty. +# Even a single row indicates completion of the data load. +table_has_rows() { + [ "$(psql_val "SELECT EXISTS (SELECT 1 FROM $1 LIMIT 1)")" = "t" ] +} + +load_data() { + local subdir table statement + while IFS=$'\t' read -r subdir table statement; do + if table_has_rows "${table}"; then + echo " ${table}: already loaded, skipping" + continue + fi + echo " ${table}: loading" + (cd "${DATA_DIR}/${subdir}" && printf '%s\n' "${statement}" | psql_run) + done < <(parse_copy_statements) +} + +add_constraints() { + psql_run -f "${SCRIPT_DIR}/constraint.sql" +} + +build_indexes() { + psql_run -f "${SCRIPT_DIR}/index.sql" +} + +make_concepts() { + # postgres-make-concepts.sql pulls in the individual concepts with \i, which + # resolves relative to the working directory rather than to the script. + cd "${CONCEPTS_DIR}" + psql_run -f postgres-make-concepts.sql +} + +validate() { + # default validate the full DB + local script=validate.sql + + # The demo is a 100 patient subset of MIMIC-IV, so it has its own set of + # expected row counts. + if [ "$(psql_val 'SELECT count(*) FROM mimiciv_hosp.patients')" -eq 100 ]; then + echo " 100 patients found, validating against the MIMIC-IV demo counts" + script=validate_demo.sql + fi + + local output + output=$(psql -v ON_ERROR_STOP=1 -f "${SCRIPT_DIR}/${script}") + echo "${output}" + + # Mismatches indicate (1) failure to load (zero rows) or (2) mismatch mimic version + if echo "${output}" | grep -q FAILED; then + echo "WARNING: some tables do not have the expected number of rows." >&2 + fi +} + +resolve_data_dir() { + local dir=${1:-${MIMIC_DATA_DIR:-}} + + if [ -z "${dir}" ]; then + usage + exit 1 + fi + if [ ! -d "${dir}/hosp" ] || [ ! -d "${dir}/icu" ]; then + echo "ERROR: ${dir} must contain the hosp/ and icu/ subfolders of MIMIC-IV." >&2 + exit 1 + fi + + DATA_DIR=$(cd -- "${dir}" && pwd) +} + +# Prefer the compressed data (the default distribution format). +select_loader() { + if [ -n "$(find "${DATA_DIR}/hosp" -maxdepth 1 -name '*.csv.gz' -print -quit)" ]; then + LOADER=load_gz.sql + elif [ -n "$(find "${DATA_DIR}/hosp" -maxdepth 1 -name '*.csv' -print -quit)" ]; then + LOADER=load.sql + else + echo "ERROR: no .csv or .csv.gz files found in ${DATA_DIR}/hosp." >&2 + exit 1 + fi + echo "Loading from ${DATA_DIR} using ${LOADER}" +} + +main() { + case "${1:-}" in + -h|--help) usage; exit 0 ;; + esac + + resolve_data_dir "${1:-}" + select_loader + + psql_run -c "CREATE TABLE IF NOT EXISTS ${PROGRESS_TABLE} ( + step text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() + )" + + run_step create create_schema + run_step load load_data + run_step constraint add_constraints + run_step index build_indexes + + if is_true "${MIMIC_MAKE_CONCEPTS:-true}"; then + run_step concepts make_concepts + else + echo "== concepts: MIMIC_MAKE_CONCEPTS is not 'true', skipping" + fi + + if is_true "${MIMIC_VALIDATE:-true}"; then + run_step validate validate + else + echo "== validate: MIMIC_VALIDATE is not 'true', skipping" + fi + + echo "MIMIC-IV build complete." +} + +main "$@" diff --git a/mimic-iv/buildmimic/postgres/docker/.env.example b/mimic-iv/buildmimic/postgres/docker/.env.example new file mode 100644 index 000000000..7750b3f91 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/.env.example @@ -0,0 +1,24 @@ +# Copy to .env and edit as needed: cp .env.example .env + +# PostgreSQL version to build against. 16, 17 and 18 are supported. +PG_VERSION=16 + +# Credentials +POSTGRES_DB=mimiciv +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres + +# Host port to publish. Change it if you already run postgres on 5432. +POSTGRES_PORT=5432 + +# Directory holding the MIMIC-IV .csv.gz files, containing hosp/ and icu/ +# subfolders. Populate it by running ./download_data.sh +MIMIC_DATA_DIR=./mimic-data + +# Derive the concepts from mimic-iv/concepts_postgres once the data is loaded. +# Set to false to load the raw tables only. +MIMIC_MAKE_CONCEPTS=true + +# Check the loaded tables against the expected row counts. The demo and the full +# dataset are detected automatically. +MIMIC_VALIDATE=true diff --git a/mimic-iv/buildmimic/postgres/docker/Dockerfile b/mimic-iv/buildmimic/postgres/docker/Dockerfile new file mode 100644 index 000000000..d7368334f --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/Dockerfile @@ -0,0 +1,18 @@ +# Image for the one-shot job that builds MIMIC-IV into the database. The +# database itself runs the stock postgres image; see docker-compose.yml. +# +# The build context is the mimic-iv/ directory, and the repository layout is +# kept intact inside the image so that build_mimic.sh finds the SQL beside it +# and the concepts two directories up. Nothing is duplicated. +# +# PostgreSQL 16, 17 and 18 are all supported. +ARG PG_VERSION=16 +FROM postgres:${PG_VERSION} + +COPY buildmimic/postgres/*.sql buildmimic/postgres/build_mimic.sh /mimic/buildmimic/postgres/ +COPY concepts_postgres/ /mimic/concepts_postgres/ + +# Replaces the postgres image's own entrypoint: this container is a psql client, +# not a server. The data directory is passed as the command. +ENTRYPOINT ["/mimic/buildmimic/postgres/build_mimic.sh"] +CMD ["/data"] diff --git a/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore b/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..e454902cd --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +# The build context is mimic-iv/, which holds large demo databases and the +# notebooks. Exclude everything, then add back only what the image needs. +* +!buildmimic/postgres/*.sql +!buildmimic/postgres/build_mimic.sh +!concepts_postgres/ diff --git a/mimic-iv/buildmimic/postgres/docker/README.md b/mimic-iv/buildmimic/postgres/docker/README.md new file mode 100644 index 000000000..2dd6b8f08 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/README.md @@ -0,0 +1,101 @@ +# MIMIC-IV with PostgreSQL and Docker + +Build a containerized PostgreSQL database containing MIMIC-IV. +The docker container uses build scripts in the parent folder. +It creates the schema, loads the data, adds the constraints and indexes, +(optionally, default true) derives the concepts from +[concepts_postgres](../../../concepts_postgres), and validates the result. + +There are two services. `mimic-db` is a stock postgres image holding the data. +`mimic-build` is a single job that populates it and then exits. They share the +PostgreSQL unix socket, so the bulk `COPY` does not cross the container network. + +## Requirements + +* Docker, with Compose v2. +* The dataset (`../../download_data.sh` is a convenience for downloading data from PhysioNet) +* Disk space. The compressed download is about 10 GB, but the loaded database is + considerably larger: roughly 140 GB for `mimiciv_hosp` and `mimiciv_icu` with + their indexes, plus about 10 GB if you build the concepts. On macOS and + Windows this space is taken from the Docker Desktop virtual disk, so raise + that limit in Settings first. + +## Quickstart + +```bash +# 1. Configure. The defaults should work as-is. +# Make sure MIMIC_DATA_DIR has the hosp/ and icu/ subfolders. +cp .env.example .env + +# 2. Download the data, if you do not already have it (about 10 GB). +../../download_data.sh + +# 3. Build. Runs in the foreground so you can watch it; add -d to detach. +docker compose up +``` + +Loading takes several hours for the full dataset, plus about an hour for the +concepts. The database accepts connections throughout, so an empty or partial +result simply means the build is still running. Watch it with: + +```bash +docker compose logs -f mimic-build +``` + +## Notes + +### `download_data.sh` + +`download_data.sh` is a convenience wrapper around `wget`. If you already have +MIMIC-IV, skip it and point `MIMIC_DATA_DIR` in `.env` at your copy. Any +directory containing the `hosp/` and `icu/` subfolders will work, including the +[demo dataset](https://physionet.org/content/mimic-iv-demo/), which is a useful +way to try this out without downloading the full 10 GB. + +### If the build is interrupted + +Run `docker compose up` again. The build records each step as it completes and +skips tables that are already populated, so it resumes rather than starting +over. This matters mostly for `chartevents`, the largest table. + +Resuming is safe because a `\COPY` is a single statement: a table is either fully loaded or empty. Progress is tracked in the +`mimiciv_build_progress` table, which you can inspect: + +```bash +docker compose exec mimic-db psql -U postgres -d mimiciv -c 'TABLE mimiciv_build_progress' +``` + +To discard everything and start clean: + +```bash +docker compose down -v +docker compose up --build +``` + +Use `--build` whenever you change the SQL or the postgres version, otherwise +Compose reuses the existing image. + +### Settings + +All are set in `.env`; see `.env.example` for the defaults. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `PG_VERSION` | `16` | PostgreSQL major version. 16, 17 and 18 are supported. | +| `POSTGRES_DB` | `mimiciv` | Database name. | +| `POSTGRES_USER` | `postgres` | Database user. | +| `POSTGRES_PASSWORD` | `postgres` | Database password. | +| `POSTGRES_PORT` | `5432` | Host port to publish. | +| `MIMIC_DATA_DIR` | `./mimic-data` | Directory holding `hosp/` and `icu/`. | +| `MIMIC_MAKE_CONCEPTS` | `true` | Derive the concepts into `mimiciv_derived`. | +| `MIMIC_VALIDATE` | `true` | Check the loaded tables against expected row counts. | + +## Using the database + +```bash +# from the host, if you have psql installed +psql -h localhost -U postgres -d mimiciv + +# otherwise, from inside the container +docker compose exec mimic-db psql -U postgres -d mimiciv +``` diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml new file mode 100644 index 000000000..4fdfa3e24 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -0,0 +1,53 @@ +name: mimic-postgres + +services: + mimic-db: + container_name: mimic-db + image: postgres:${PG_VERSION:-16} + ports: + - "${POSTGRES_PORT:-5432}:5432" + environment: + POSTGRES_DB: ${POSTGRES_DB:-mimiciv} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + volumes: + # Mounted at /var/lib/postgresql rather than the data directory itself. + # PostgreSQL 18 moved the data directory into a per-version subfolder. + - mimic-pgdata:/var/lib/postgresql + # Shares the unix socket with mimic-build, so the bulk COPY does not have + # to cross the container network. + - mimic-socket:/var/run/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 + + # Loads the data and derives the concepts, then exits. + # Re-running it resumes an interrupted build. + mimic-build: + build: + # mimic-iv/, so the build can reach buildmimic/postgres and concepts_postgres + context: ../../.. + dockerfile: buildmimic/postgres/docker/Dockerfile + args: + PG_VERSION: ${PG_VERSION:-16} + depends_on: + mimic-db: + condition: service_healthy + environment: + # Connects over the shared unix socket rather than TCP. + PGHOST: /var/run/postgresql + PGDATABASE: ${POSTGRES_DB:-mimiciv} + PGUSER: ${POSTGRES_USER:-postgres} + PGPASSWORD: ${POSTGRES_PASSWORD:-postgres} + MIMIC_MAKE_CONCEPTS: ${MIMIC_MAKE_CONCEPTS:-true} + MIMIC_VALIDATE: ${MIMIC_VALIDATE:-true} + volumes: + # The .csv.gz files, read-only. Must contain hosp/ and icu/ subfolders. + - ${MIMIC_DATA_DIR:-./mimic-data}:/data:ro + - mimic-socket:/var/run/postgresql + +volumes: + mimic-pgdata: + mimic-socket: