From e472f9c126ad534db595aaec40dd4bfda6f2f430 Mon Sep 17 00:00:00 2001 From: mlbernauer Date: Fri, 14 Jun 2024 22:16:13 -0600 Subject: [PATCH 1/8] added scripts for dockerizing mimic-iv with postgres --- .../buildmimic/postgres/docker/Dockerfile | 4 + mimic-iv/buildmimic/postgres/docker/README.md | 37 ++ .../postgres/docker/docker-compose.yml | 30 + .../postgres/docker/download_data.sh | 4 + .../postgres/docker/src/01_create.sql | 519 +++++++++++++++++ .../postgres/docker/src/02_load_gz.sql | 37 ++ .../postgres/docker/src/03_constrant.sql | 551 ++++++++++++++++++ .../postgres/docker/src/04_index.sql | 232 ++++++++ 8 files changed, 1414 insertions(+) create mode 100644 mimic-iv/buildmimic/postgres/docker/Dockerfile create mode 100644 mimic-iv/buildmimic/postgres/docker/README.md create mode 100644 mimic-iv/buildmimic/postgres/docker/docker-compose.yml create mode 100755 mimic-iv/buildmimic/postgres/docker/download_data.sh create mode 100644 mimic-iv/buildmimic/postgres/docker/src/01_create.sql create mode 100644 mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql create mode 100644 mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql create mode 100644 mimic-iv/buildmimic/postgres/docker/src/04_index.sql diff --git a/mimic-iv/buildmimic/postgres/docker/Dockerfile b/mimic-iv/buildmimic/postgres/docker/Dockerfile new file mode 100644 index 000000000..2dd098fb3 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/Dockerfile @@ -0,0 +1,4 @@ +FROM postgres +COPY ./src/ /docker-entrypoint-initdb.d/ +WORKDIR /app +EXPOSE 5432 diff --git a/mimic-iv/buildmimic/postgres/docker/README.md b/mimic-iv/buildmimic/postgres/docker/README.md new file mode 100644 index 000000000..242cdeb7a --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/README.md @@ -0,0 +1,37 @@ +# Getting MIMIC-IV up and running with Postgres and Docker +Use these scripts to quickly get setup with a containerized version of MIMIC-IV + +## Steps +1. Install Docker +2. Download mimic data from PhysioNet by running the `download_data.sh` script. this will store the files in ./mimic-data/ +3. Edit `POSTGRES_USER` and `POSTGRES_PASSWORD` as needed. Optionally, you can + change the host paths for the volume mounts if you would like to change where + the data are stored on the host machine. +4. Build and run the container by running `docker compose up`. This may take awhile. + +Once complete you should have a containerized postgres instance containing MIMIC-IV data in the `mimiciv` database. + +## Using the database + +```bash +# spin up a container and detach +docker compose up -d + +# you should now see a running container...e.g docker-mimic-db-1 +docker ps + +# if you have psql installed on the host machine you can do...password required +psql -U -d mimiciv -h localhost + +# if you do not have psql installed on host machine you can do...password not +# required +docker exec -it docker-mimic-db-1 psql -U -d mimiciv + +# psql on host machine...run profile script +psql -U -d mimiciv -h localhost -f ../validate.sql + +# psql not on host machine...need to mv validate into mount director so it can +# be accessed from container +cp ../validate.sql ./mimic-data/ +docker exec -it docker-mimic-db-1 psql -U -d mimiciv -f /data/validate.sql +``` diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml new file mode 100644 index 000000000..b1e9c4ced --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -0,0 +1,30 @@ +# Use postgres/example user/password credentials +version: '3.9' +services: + mimic-db: + image: mimiciv + build: . + restart: always + ports: + # you may need to change the host port if you have another postgres + # instance running on the host machine. + - 5432:5432 + volumes: + # this mounts the mimic csv files to the /data/ directory within the + # container. + - ./mimic-data/:/data/ + # this maps a directory on the host machine (i.e. ./mimic-db/) to the + # container so that the data in the database can persist after container + # has been shut down. + - ./mimic-db/:/var/lib/postgresql/data/ + environment: + # this will be the name of the database within Postgres. + POSTGRES_DB: mimiciv + # set this to the same user that owns the directory storing the .csv data + # (hint: check ls -l ./mimic-data/) this should be the user of host + # machine. this will also be the user you will use to authenticate with + # postgres + POSTGRES_USER: mlbernauer + # this will be the password for authenticating with postgres...set to + # whatever you want + POSTGRES_PASSWORD: pass123 diff --git a/mimic-iv/buildmimic/postgres/docker/download_data.sh b/mimic-iv/buildmimic/postgres/docker/download_data.sh new file mode 100755 index 000000000..0819330bb --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/download_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +read -p "Enter PhysioNet username: " USERNAME +echo "Downloading MIMIV-IV data to ./mimic-data/" +wget -P -A gz,txt ./mimic-data/ -r -N -c -nd -np --user ${USERNAME} --ask-password https://physionet.org/files/mimiciv/2.2/ diff --git a/mimic-iv/buildmimic/postgres/docker/src/01_create.sql b/mimic-iv/buildmimic/postgres/docker/src/01_create.sql new file mode 100644 index 000000000..14080d22a --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/src/01_create.sql @@ -0,0 +1,519 @@ +------------------------------------------- +-- Create the tables and MIMIC-IV schema -- +------------------------------------------- + +---------------------- +-- Creating schemas -- +---------------------- + +DROP SCHEMA IF EXISTS mimiciv_hosp CASCADE; +CREATE SCHEMA mimiciv_hosp; +DROP SCHEMA IF EXISTS mimiciv_icu CASCADE; +CREATE SCHEMA mimiciv_icu; +DROP SCHEMA IF EXISTS mimiciv_derived CASCADE; +CREATE SCHEMA mimiciv_derived; + +--------------------- +-- Creating tables -- +--------------------- + +-- hosp schema + +DROP TABLE IF EXISTS mimiciv_hosp.admissions; +CREATE TABLE mimiciv_hosp.admissions +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + admittime TIMESTAMP NOT NULL, + dischtime TIMESTAMP, + deathtime TIMESTAMP, + admission_type VARCHAR(40) NOT NULL, + admit_provider_id VARCHAR(10), + admission_location VARCHAR(60), + discharge_location VARCHAR(60), + insurance VARCHAR(255), + language VARCHAR(10), + marital_status VARCHAR(30), + race VARCHAR(80), + edregtime TIMESTAMP, + edouttime TIMESTAMP, + hospital_expire_flag SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_hosp.d_hcpcs; +CREATE TABLE mimiciv_hosp.d_hcpcs +( + code CHAR(5) NOT NULL, + category SMALLINT, + long_description TEXT, + short_description VARCHAR(180) +); + +DROP TABLE IF EXISTS mimiciv_hosp.diagnoses_icd; +CREATE TABLE mimiciv_hosp.diagnoses_icd +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + icd_code CHAR(7), + icd_version SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_hosp.d_icd_diagnoses; +CREATE TABLE mimiciv_hosp.d_icd_diagnoses +( + icd_code CHAR(7) NOT NULL, + icd_version SMALLINT NOT NULL, + long_title VARCHAR(255) +); + +DROP TABLE IF EXISTS mimiciv_hosp.d_icd_procedures; +CREATE TABLE mimiciv_hosp.d_icd_procedures +( + icd_code CHAR(7) NOT NULL, + icd_version SMALLINT NOT NULL, + long_title VARCHAR(222) +); + +DROP TABLE IF EXISTS mimiciv_hosp.d_labitems; +CREATE TABLE mimiciv_hosp.d_labitems +( + itemid INTEGER NOT NULL, + label VARCHAR(50), + fluid VARCHAR(50), + category VARCHAR(50) +); + +DROP TABLE IF EXISTS mimiciv_hosp.drgcodes; +CREATE TABLE mimiciv_hosp.drgcodes +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + drg_type VARCHAR(4), + drg_code VARCHAR(10) NOT NULL, + description VARCHAR(195), + drg_severity SMALLINT, + drg_mortality SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_hosp.emar_detail; +CREATE TABLE mimiciv_hosp.emar_detail +( + subject_id INTEGER NOT NULL, + emar_id VARCHAR(25) NOT NULL, + emar_seq INTEGER NOT NULL, + parent_field_ordinal VARCHAR(10), + administration_type VARCHAR(50), + pharmacy_id INTEGER, + barcode_type VARCHAR(4), + reason_for_no_barcode TEXT, + complete_dose_not_given VARCHAR(5), + dose_due VARCHAR(100), + dose_due_unit VARCHAR(50), + dose_given VARCHAR(255), + dose_given_unit VARCHAR(50), + will_remainder_of_dose_be_given VARCHAR(5), + product_amount_given VARCHAR(30), + product_unit VARCHAR(30), + product_code VARCHAR(30), + product_description VARCHAR(255), + product_description_other VARCHAR(255), + prior_infusion_rate VARCHAR(40), + infusion_rate VARCHAR(40), + infusion_rate_adjustment VARCHAR(50), + infusion_rate_adjustment_amount VARCHAR(30), + infusion_rate_unit VARCHAR(30), + route VARCHAR(10), + infusion_complete VARCHAR(1), + completion_interval VARCHAR(50), + new_iv_bag_hung VARCHAR(1), + continued_infusion_in_other_location VARCHAR(1), + restart_interval VARCHAR(2305), + side VARCHAR(10), + site VARCHAR(255), + non_formulary_visual_verification VARCHAR(1) +); + +DROP TABLE IF EXISTS mimiciv_hosp.emar; +CREATE TABLE mimiciv_hosp.emar +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER, + emar_id VARCHAR(25) NOT NULL, + emar_seq INTEGER NOT NULL, + poe_id VARCHAR(25) NOT NULL, + pharmacy_id INTEGER, + enter_provider_id VARCHAR(10), + charttime TIMESTAMP NOT NULL, + medication TEXT, + event_txt VARCHAR(100), + scheduletime TIMESTAMP, + storetime TIMESTAMP NOT NULL +); + +DROP TABLE IF EXISTS mimiciv_hosp.hcpcsevents; +CREATE TABLE mimiciv_hosp.hcpcsevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + chartdate DATE, + hcpcs_cd CHAR(5) NOT NULL, + seq_num INTEGER NOT NULL, + short_description VARCHAR(180) +); + +DROP TABLE IF EXISTS mimiciv_hosp.labevents; +CREATE TABLE mimiciv_hosp.labevents +( + labevent_id INTEGER NOT NULL, + subject_id INTEGER NOT NULL, + hadm_id INTEGER, + specimen_id INTEGER NOT NULL, + itemid INTEGER NOT NULL, + order_provider_id VARCHAR(10), + charttime TIMESTAMP(0), + storetime TIMESTAMP(0), + value VARCHAR(200), + valuenum DOUBLE PRECISION, + valueuom VARCHAR(20), + ref_range_lower DOUBLE PRECISION, + ref_range_upper DOUBLE PRECISION, + flag VARCHAR(10), + priority VARCHAR(7), + comments TEXT +); + +DROP TABLE IF EXISTS mimiciv_hosp.microbiologyevents; +CREATE TABLE mimiciv_hosp.microbiologyevents +( + microevent_id INTEGER NOT NULL, + subject_id INTEGER NOT NULL, + hadm_id INTEGER, + micro_specimen_id INTEGER NOT NULL, + order_provider_id VARCHAR(10), + chartdate TIMESTAMP(0) NOT NULL, + charttime TIMESTAMP(0), + spec_itemid INTEGER NOT NULL, + spec_type_desc VARCHAR(100) NOT NULL, + test_seq INTEGER NOT NULL, + storedate TIMESTAMP(0), + storetime TIMESTAMP(0), + test_itemid INTEGER, + test_name VARCHAR(100), + org_itemid INTEGER, + org_name VARCHAR(100), + isolate_num SMALLINT, + quantity VARCHAR(50), + ab_itemid INTEGER, + ab_name VARCHAR(30), + dilution_text VARCHAR(10), + dilution_comparison VARCHAR(20), + dilution_value DOUBLE PRECISION, + interpretation VARCHAR(5), + comments TEXT +); + +DROP TABLE IF EXISTS mimiciv_hosp.omr; +CREATE TABLE mimiciv_hosp.omr +( + subject_id INTEGER NOT NULL, + chartdate DATE NOT NULL, + seq_num INTEGER NOT NULL, + result_name VARCHAR(100) NOT NULL, + result_value TEXT NOT NULL +); + +DROP TABLE IF EXISTS mimiciv_hosp.patients; +CREATE TABLE mimiciv_hosp.patients +( + subject_id INTEGER NOT NULL, + gender CHAR(1) NOT NULL, + anchor_age SMALLINT, + anchor_year SMALLINT NOT NULL, + anchor_year_group VARCHAR(20) NOT NULL, + dod DATE +); + +DROP TABLE IF EXISTS mimiciv_hosp.pharmacy; +CREATE TABLE mimiciv_hosp.pharmacy +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + pharmacy_id INTEGER NOT NULL, + poe_id VARCHAR(25), + starttime TIMESTAMP(3), + stoptime TIMESTAMP(3), + medication TEXT, + proc_type VARCHAR(50) NOT NULL, + status VARCHAR(50), + entertime TIMESTAMP(3) NOT NULL, + verifiedtime TIMESTAMP(3), + route VARCHAR(50), + frequency VARCHAR(50), + disp_sched VARCHAR(255), + infusion_type VARCHAR(15), + sliding_scale VARCHAR(1), + lockout_interval VARCHAR(50), + basal_rate REAL, + one_hr_max VARCHAR(10), + doses_per_24_hrs REAL, + duration REAL, + duration_interval VARCHAR(50), + expiration_value INTEGER, + expiration_unit VARCHAR(50), + expirationdate TIMESTAMP(3), + dispensation VARCHAR(50), + fill_quantity VARCHAR(50) +); + +DROP TABLE IF EXISTS mimiciv_hosp.poe_detail; +CREATE TABLE mimiciv_hosp.poe_detail +( + poe_id VARCHAR(25) NOT NULL, + poe_seq INTEGER NOT NULL, + subject_id INTEGER NOT NULL, + field_name VARCHAR(255) NOT NULL, + field_value TEXT +); + +DROP TABLE IF EXISTS mimiciv_hosp.poe; +CREATE TABLE mimiciv_hosp.poe +( + poe_id VARCHAR(25) NOT NULL, + poe_seq INTEGER NOT NULL, + subject_id INTEGER NOT NULL, + hadm_id INTEGER, + ordertime TIMESTAMP(0) NOT NULL, + order_type VARCHAR(25) NOT NULL, + order_subtype VARCHAR(50), + transaction_type VARCHAR(15), + discontinue_of_poe_id VARCHAR(25), + discontinued_by_poe_id VARCHAR(25), + order_provider_id VARCHAR(10), + order_status VARCHAR(15) +); + +DROP TABLE IF EXISTS mimiciv_hosp.prescriptions; +CREATE TABLE mimiciv_hosp.prescriptions +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + pharmacy_id INTEGER NOT NULL, + poe_id VARCHAR(25), + poe_seq INTEGER, + order_provider_id VARCHAR(10), + starttime TIMESTAMP(3), + stoptime TIMESTAMP(3), + drug_type VARCHAR(20) NOT NULL, + drug VARCHAR(255) NOT NULL, + formulary_drug_cd VARCHAR(50), + gsn VARCHAR(255), + ndc VARCHAR(25), + prod_strength VARCHAR(255), + form_rx VARCHAR(25), + dose_val_rx VARCHAR(100), + dose_unit_rx VARCHAR(50), + form_val_disp VARCHAR(50), + form_unit_disp VARCHAR(50), + doses_per_24_hrs REAL, + route VARCHAR(50) +); + +DROP TABLE IF EXISTS mimiciv_hosp.procedures_icd; +CREATE TABLE mimiciv_hosp.procedures_icd +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + chartdate DATE NOT NULL, + icd_code VARCHAR(7), + icd_version SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_hosp.provider; +CREATE TABLE mimiciv_hosp.provider +( + provider_id VARCHAR(10) NOT NULL +); + +DROP TABLE IF EXISTS mimiciv_hosp.services; +CREATE TABLE mimiciv_hosp.services +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + transfertime TIMESTAMP NOT NULL, + prev_service VARCHAR(10), + curr_service VARCHAR(10) +); + +DROP TABLE IF EXISTS mimiciv_hosp.transfers; +CREATE TABLE mimiciv_hosp.transfers +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER, + transfer_id INTEGER NOT NULL, + eventtype VARCHAR(10), + careunit VARCHAR(255), + intime TIMESTAMP, + outtime TIMESTAMP +); + +-- icu schema + +DROP TABLE IF EXISTS mimiciv_icu.caregiver; +CREATE TABLE mimiciv_icu.caregiver +( + caregiver_id INTEGER NOT NULL +); + +DROP TABLE IF EXISTS mimiciv_icu.chartevents; +CREATE TABLE mimiciv_icu.chartevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER NOT NULL, + caregiver_id INTEGER, + charttime TIMESTAMP NOT NULL, + storetime TIMESTAMP, + itemid INTEGER NOT NULL, + value VARCHAR(200), + valuenum FLOAT, + valueuom VARCHAR(20), + warning SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_icu.datetimeevents; +CREATE TABLE mimiciv_icu.datetimeevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER NOT NULL, + caregiver_id INTEGER, + charttime TIMESTAMP NOT NULL, + storetime TIMESTAMP, + itemid INTEGER NOT NULL, + value TIMESTAMP NOT NULL, + valueuom VARCHAR(20), + warning SMALLINT +); + +DROP TABLE IF EXISTS mimiciv_icu.d_items; +CREATE TABLE mimiciv_icu.d_items +( + itemid INTEGER NOT NULL, + label VARCHAR(100) NOT NULL, + abbreviation VARCHAR(50) NOT NULL, + linksto VARCHAR(30) NOT NULL, + category VARCHAR(50) NOT NULL, + unitname VARCHAR(50), + param_type VARCHAR(20) NOT NULL, + lownormalvalue FLOAT, + highnormalvalue FLOAT +); + +DROP TABLE IF EXISTS mimiciv_icu.icustays; +CREATE TABLE mimiciv_icu.icustays +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER NOT NULL, + first_careunit VARCHAR(255), + last_careunit VARCHAR(255), + intime TIMESTAMP, + outtime TIMESTAMP, + los FLOAT +); + +DROP TABLE IF EXISTS mimiciv_icu.ingredientevents; +CREATE TABLE mimiciv_icu.ingredientevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER, + caregiver_id INTEGER, + starttime TIMESTAMP NOT NULL, + endtime TIMESTAMP NOT NULL, + storetime TIMESTAMP, + itemid INTEGER NOT NULL, + amount FLOAT, + amountuom VARCHAR(20), + rate FLOAT, + rateuom VARCHAR(20), + orderid INTEGER NOT NULL, + linkorderid INTEGER, + statusdescription VARCHAR(20), + originalamount FLOAT, + originalrate FLOAT +); + +DROP TABLE IF EXISTS mimiciv_icu.inputevents; +CREATE TABLE mimiciv_icu.inputevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER, + caregiver_id INTEGER, + starttime TIMESTAMP NOT NULL, + endtime TIMESTAMP NOT NULL, + storetime TIMESTAMP, + itemid INTEGER NOT NULL, + amount FLOAT, + amountuom VARCHAR(20), + rate FLOAT, + rateuom VARCHAR(20), + orderid INTEGER NOT NULL, + linkorderid INTEGER, + ordercategoryname VARCHAR(50), + secondaryordercategoryname VARCHAR(50), + ordercomponenttypedescription VARCHAR(100), + ordercategorydescription VARCHAR(30), + patientweight FLOAT, + totalamount FLOAT, + totalamountuom VARCHAR(50), + isopenbag SMALLINT, + continueinnextdept SMALLINT, + statusdescription VARCHAR(20), + originalamount FLOAT, + originalrate FLOAT +); + +DROP TABLE IF EXISTS mimiciv_icu.outputevents; +CREATE TABLE mimiciv_icu.outputevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER NOT NULL, + caregiver_id INTEGER, + charttime TIMESTAMP(3) NOT NULL, + storetime TIMESTAMP(3) NOT NULL, + itemid INTEGER NOT NULL, + value FLOAT NOT NULL, + valueuom VARCHAR(20) +); + +DROP TABLE IF EXISTS mimiciv_icu.procedureevents; +CREATE TABLE mimiciv_icu.procedureevents +( + subject_id INTEGER NOT NULL, + hadm_id INTEGER NOT NULL, + stay_id INTEGER NOT NULL, + caregiver_id INTEGER, + starttime TIMESTAMP NOT NULL, + endtime TIMESTAMP NOT NULL, + storetime TIMESTAMP NOT NULL, + itemid INTEGER NOT NULL, + value FLOAT, + valueuom VARCHAR(20), + location VARCHAR(100), + locationcategory VARCHAR(50), + orderid INTEGER, + linkorderid INTEGER, + ordercategoryname VARCHAR(50), + ordercategorydescription VARCHAR(30), + patientweight FLOAT, + isopenbag SMALLINT, + continueinnextdept SMALLINT, + statusdescription VARCHAR(20), + originalamount FLOAT, + originalrate FLOAT +); diff --git a/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql b/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql new file mode 100644 index 000000000..01bff843e --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql @@ -0,0 +1,37 @@ +----------------------------------------- +-- Load data into the MIMIC-IV schemas -- +----------------------------------------- + +SET CLIENT_ENCODING TO 'utf8'; +\cd /data +\COPY mimiciv_hosp.admissions FROM PROGRAM 'gzip -dc admissions.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.d_hcpcs FROM PROGRAM 'gzip -dc d_hcpcs.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.diagnoses_icd FROM PROGRAM 'gzip -dc diagnoses_icd.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.d_icd_diagnoses FROM PROGRAM 'gzip -dc d_icd_diagnoses.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.d_icd_procedures FROM PROGRAM 'gzip -dc d_icd_procedures.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.d_labitems FROM PROGRAM 'gzip -dc d_labitems.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.drgcodes FROM PROGRAM 'gzip -dc drgcodes.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.emar_detail FROM PROGRAM 'gzip -dc emar_detail.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.emar FROM PROGRAM 'gzip -dc emar.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.hcpcsevents FROM PROGRAM 'gzip -dc hcpcsevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.labevents FROM PROGRAM 'gzip -dc labevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.microbiologyevents FROM PROGRAM 'gzip -dc microbiologyevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.omr FROM PROGRAM 'gzip -dc omr.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.patients FROM PROGRAM 'gzip -dc patients.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.pharmacy FROM PROGRAM 'gzip -dc pharmacy.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.poe_detail FROM PROGRAM 'gzip -dc poe_detail.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.poe FROM PROGRAM 'gzip -dc poe.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.prescriptions FROM PROGRAM 'gzip -dc prescriptions.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.procedures_icd FROM PROGRAM 'gzip -dc procedures_icd.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.provider FROM PROGRAM 'gzip -dc provider.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.services FROM PROGRAM 'gzip -dc services.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_hosp.transfers FROM PROGRAM 'gzip -dc transfers.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.caregiver FROM PROGRAM 'gzip -dc caregiver.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.chartevents FROM PROGRAM 'gzip -dc chartevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.datetimeevents FROM PROGRAM 'gzip -dc datetimeevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.d_items FROM PROGRAM 'gzip -dc d_items.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.icustays FROM PROGRAM 'gzip -dc icustays.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.ingredientevents FROM PROGRAM 'gzip -dc ingredientevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.inputevents FROM PROGRAM 'gzip -dc inputevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.outputevents FROM PROGRAM 'gzip -dc outputevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; +\COPY mimiciv_icu.procedureevents FROM PROGRAM 'gzip -dc procedureevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; diff --git a/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql b/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql new file mode 100644 index 000000000..6c63c2efd --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql @@ -0,0 +1,551 @@ +--------------------------- +--------------------------- +-- Creating Primary Keys -- +--------------------------- +--------------------------- + +---------- +-- hosp -- +---------- + +-- admissions + +ALTER TABLE mimiciv_hosp.admissions DROP CONSTRAINT IF EXISTS admissions_pk CASCADE; +ALTER TABLE mimiciv_hosp.admissions +ADD CONSTRAINT admissions_pk + PRIMARY KEY (hadm_id); + +-- d_hcpcs + +ALTER TABLE mimiciv_hosp.d_hcpcs DROP CONSTRAINT IF EXISTS d_hcpcs_pk CASCADE; +ALTER TABLE mimiciv_hosp.d_hcpcs +ADD CONSTRAINT d_hcpcs_pk + PRIMARY KEY (code); + +-- diagnoses_icd + +ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_pk CASCADE; +ALTER TABLE mimiciv_hosp.diagnoses_icd +ADD CONSTRAINT diagnoses_icd_pk + PRIMARY KEY (hadm_id, seq_num, icd_code, icd_version); + +-- d_icd_diagnoses + +ALTER TABLE mimiciv_hosp.d_icd_diagnoses DROP CONSTRAINT IF EXISTS d_icd_diagnoses_pk CASCADE; +ALTER TABLE mimiciv_hosp.d_icd_diagnoses +ADD CONSTRAINT d_icd_diagnoses_pk + PRIMARY KEY (icd_code, icd_version); + +-- d_icd_procedures + +ALTER TABLE mimiciv_hosp.d_icd_procedures DROP CONSTRAINT IF EXISTS d_icd_procedures_pk CASCADE; +ALTER TABLE mimiciv_hosp.d_icd_procedures +ADD CONSTRAINT d_icd_procedures_pk + PRIMARY KEY (icd_code, icd_version); + +-- d_labitems + +ALTER TABLE mimiciv_hosp.d_labitems DROP CONSTRAINT IF EXISTS d_labitems_pk CASCADE; +ALTER TABLE mimiciv_hosp.d_labitems +ADD CONSTRAINT d_labitems_pk + PRIMARY KEY (itemid); + +-- emar_detail + +-- ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_pk; +-- ALTER TABLE mimiciv_hosp.emar_detail +-- ADD CONSTRAINT emar_detail_pk +-- PRIMARY KEY (emar_id, parent_field_ordinal); + +-- emar + +ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_pk CASCADE; +ALTER TABLE mimiciv_hosp.emar +ADD CONSTRAINT emar_pk + PRIMARY KEY (emar_id); + +-- hcpcsevents + +ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_pk CASCADE; +ALTER TABLE mimiciv_hosp.hcpcsevents +ADD CONSTRAINT hcpcsevents_pk + PRIMARY KEY (hadm_id, hcpcs_cd, seq_num); + +-- labevents + +ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_pk CASCADE; +ALTER TABLE mimiciv_hosp.labevents +ADD CONSTRAINT labevents_pk + PRIMARY KEY (labevent_id); + +-- microbiologyevents + +ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_pk CASCADE; +ALTER TABLE mimiciv_hosp.microbiologyevents +ADD CONSTRAINT microbiologyevents_pk + PRIMARY KEY (microevent_id); + +-- patients + +ALTER TABLE mimiciv_hosp.patients DROP CONSTRAINT IF EXISTS patients_pk CASCADE; +ALTER TABLE mimiciv_hosp.patients +ADD CONSTRAINT patients_pk + PRIMARY KEY (subject_id); + +-- pharmacy + +ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_pk CASCADE; +ALTER TABLE mimiciv_hosp.pharmacy +ADD CONSTRAINT pharmacy_pk + PRIMARY KEY (pharmacy_id); + +-- poe_detail + +ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_pk CASCADE; +ALTER TABLE mimiciv_hosp.poe_detail +ADD CONSTRAINT poe_detail_pk + PRIMARY KEY (poe_id, field_name); + +-- poe + +ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_pk CASCADE; +ALTER TABLE mimiciv_hosp.poe +ADD CONSTRAINT poe_pk + PRIMARY KEY (poe_id); + +-- prescriptions + +ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_pk CASCADE; +ALTER TABLE mimiciv_hosp.prescriptions +ADD CONSTRAINT prescriptions_pk + PRIMARY KEY (pharmacy_id, drug_type, drug); + +-- procedures_icd + +ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_pk CASCADE; +ALTER TABLE mimiciv_hosp.procedures_icd +ADD CONSTRAINT procedures_icd_pk + PRIMARY KEY (hadm_id, seq_num, icd_code, icd_version); + +-- services + +ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_pk CASCADE; +ALTER TABLE mimiciv_hosp.services +ADD CONSTRAINT services_pk + PRIMARY KEY (hadm_id, transfertime, curr_service); + +--------- +-- icu -- +--------- + +-- datetimeevents + +ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_pk CASCADE; +ALTER TABLE mimiciv_icu.datetimeevents +ADD CONSTRAINT datetimeevents_pk + PRIMARY KEY (stay_id, itemid, charttime); + +-- d_items + +ALTER TABLE mimiciv_icu.d_items DROP CONSTRAINT IF EXISTS d_items_pk CASCADE; +ALTER TABLE mimiciv_icu.d_items +ADD CONSTRAINT d_items_pk + PRIMARY KEY (itemid); + +-- icustays + +ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_pk CASCADE; +ALTER TABLE mimiciv_icu.icustays +ADD CONSTRAINT icustays_pk + PRIMARY KEY (stay_id); + +-- inputevents + +ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_pk CASCADE; +ALTER TABLE mimiciv_icu.inputevents +ADD CONSTRAINT inputevents_pk + PRIMARY KEY (orderid, itemid); + +-- outputevents + +ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_pk CASCADE; +ALTER TABLE mimiciv_icu.outputevents +ADD CONSTRAINT outputevents_pk + PRIMARY KEY (stay_id, charttime, itemid); + +-- procedureevents + +ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_pk CASCADE; +ALTER TABLE mimiciv_icu.procedureevents +ADD CONSTRAINT procedureevents_pk + PRIMARY KEY (orderid); + +--------------------------- +--------------------------- +-- Creating Foreign Keys -- +--------------------------- +--------------------------- + +---------- +-- hosp -- +---------- + +-- admissions + +ALTER TABLE mimiciv_hosp.admissions DROP CONSTRAINT IF EXISTS admissions_patients_fk; +ALTER TABLE mimiciv_hosp.admissions +ADD CONSTRAINT admissions_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +-- diagnoses_icd + +ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_patients_fk; +ALTER TABLE mimiciv_hosp.diagnoses_icd +ADD CONSTRAINT diagnoses_icd_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_admissions_fk; +ALTER TABLE mimiciv_hosp.diagnoses_icd +ADD CONSTRAINT diagnoses_icd_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- drgcodes + +ALTER TABLE mimiciv_hosp.drgcodes DROP CONSTRAINT IF EXISTS drgcodes_patients_fk; +ALTER TABLE mimiciv_hosp.drgcodes +ADD CONSTRAINT drgcodes_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.drgcodes DROP CONSTRAINT IF EXISTS drgcodes_admissions_fk; +ALTER TABLE mimiciv_hosp.drgcodes +ADD CONSTRAINT drgcodes_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- emar_detail + +ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_patients_fk; +ALTER TABLE mimiciv_hosp.emar_detail +ADD CONSTRAINT emar_detail_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_emar_fk; +ALTER TABLE mimiciv_hosp.emar_detail +ADD CONSTRAINT emar_detail_emar_fk + FOREIGN KEY (emar_id) + REFERENCES mimiciv_hosp.emar (emar_id); + +-- emar + +ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_patients_fk; +ALTER TABLE mimiciv_hosp.emar +ADD CONSTRAINT emar_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_admissions_fk; +ALTER TABLE mimiciv_hosp.emar +ADD CONSTRAINT emar_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- hcpcsevents + +ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_patients_fk; +ALTER TABLE mimiciv_hosp.hcpcsevents +ADD CONSTRAINT hcpcsevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_admissions_fk; +ALTER TABLE mimiciv_hosp.hcpcsevents +ADD CONSTRAINT hcpcsevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_d_hcpcs_fk; +ALTER TABLE mimiciv_hosp.hcpcsevents +ADD CONSTRAINT hcpcsevents_d_hcpcs_fk + FOREIGN KEY (hcpcs_cd) + REFERENCES mimiciv_hosp.d_hcpcs (code); + +-- labevents + +ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_patients_fk; +ALTER TABLE mimiciv_hosp.labevents +ADD CONSTRAINT labevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_d_labitems_fk; +ALTER TABLE mimiciv_hosp.labevents +ADD CONSTRAINT labevents_d_labitems_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_hosp.d_labitems (itemid); + +-- microbiologyevents + +ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_patients_fk; +ALTER TABLE mimiciv_hosp.microbiologyevents +ADD CONSTRAINT microbiologyevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_admissions_fk; +ALTER TABLE mimiciv_hosp.microbiologyevents +ADD CONSTRAINT microbiologyevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- pharmacy + +ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_patients_fk; +ALTER TABLE mimiciv_hosp.pharmacy +ADD CONSTRAINT pharmacy_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_admissions_fk; +ALTER TABLE mimiciv_hosp.pharmacy +ADD CONSTRAINT pharmacy_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- poe_detail + +ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_patients_fk; +ALTER TABLE mimiciv_hosp.poe_detail +ADD CONSTRAINT poe_detail_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_poe_fk; +ALTER TABLE mimiciv_hosp.poe_detail +ADD CONSTRAINT poe_detail_poe_fk + FOREIGN KEY (poe_id) + REFERENCES mimiciv_hosp.poe (poe_id); + +-- poe + +ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_patients_fk; +ALTER TABLE mimiciv_hosp.poe +ADD CONSTRAINT poe_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_admissions_fk; +ALTER TABLE mimiciv_hosp.poe +ADD CONSTRAINT poe_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- prescriptions + +ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_patients_fk; +ALTER TABLE mimiciv_hosp.prescriptions +ADD CONSTRAINT prescriptions_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_admissions_fk; +ALTER TABLE mimiciv_hosp.prescriptions +ADD CONSTRAINT prescriptions_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- procedures_icd + +ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_patients_fk; +ALTER TABLE mimiciv_hosp.procedures_icd +ADD CONSTRAINT procedures_icd_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_admissions_fk; +ALTER TABLE mimiciv_hosp.procedures_icd +ADD CONSTRAINT procedures_icd_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- services + +ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_patients_fk; +ALTER TABLE mimiciv_hosp.services +ADD CONSTRAINT services_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_admissions_fk; +ALTER TABLE mimiciv_hosp.services +ADD CONSTRAINT services_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- transfers + +ALTER TABLE mimiciv_hosp.transfers DROP CONSTRAINT IF EXISTS transfers_pk CASCADE; +ALTER TABLE mimiciv_hosp.transfers +ADD CONSTRAINT transfers_pk + PRIMARY KEY (transfer_id); + +-- transfers + +ALTER TABLE mimiciv_hosp.transfers DROP CONSTRAINT IF EXISTS transfers_patients_fk; +ALTER TABLE mimiciv_hosp.transfers +ADD CONSTRAINT transfers_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + + +--------- +-- icu -- +--------- + +-- chartevents + +ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_patients_fk; +ALTER TABLE mimiciv_icu.chartevents +ADD CONSTRAINT chartevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_admissions_fk; +ALTER TABLE mimiciv_icu.chartevents +ADD CONSTRAINT chartevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_icustays_fk; +ALTER TABLE mimiciv_icu.chartevents +ADD CONSTRAINT chartevents_icustays_fk + FOREIGN KEY (stay_id) + REFERENCES mimiciv_icu.icustays (stay_id); + +ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_d_items_fk; +ALTER TABLE mimiciv_icu.chartevents +ADD CONSTRAINT chartevents_d_items_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_icu.d_items (itemid); + +-- datetimeevents + +ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_patients_fk; +ALTER TABLE mimiciv_icu.datetimeevents +ADD CONSTRAINT datetimeevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_admissions_fk; +ALTER TABLE mimiciv_icu.datetimeevents +ADD CONSTRAINT datetimeevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_icustays_fk; +ALTER TABLE mimiciv_icu.datetimeevents +ADD CONSTRAINT datetimeevents_icustays_fk + FOREIGN KEY (stay_id) + REFERENCES mimiciv_icu.icustays (stay_id); + +ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_d_items_fk; +ALTER TABLE mimiciv_icu.datetimeevents +ADD CONSTRAINT datetimeevents_d_items_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_icu.d_items (itemid); + +-- icustays + +ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_patients_fk; +ALTER TABLE mimiciv_icu.icustays +ADD CONSTRAINT icustays_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_admissions_fk; +ALTER TABLE mimiciv_icu.icustays +ADD CONSTRAINT icustays_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +-- inputevents + +ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_patients_fk; +ALTER TABLE mimiciv_icu.inputevents +ADD CONSTRAINT inputevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_admissions_fk; +ALTER TABLE mimiciv_icu.inputevents +ADD CONSTRAINT inputevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_icustays_fk; +ALTER TABLE mimiciv_icu.inputevents +ADD CONSTRAINT inputevents_icustays_fk + FOREIGN KEY (stay_id) + REFERENCES mimiciv_icu.icustays (stay_id); + +ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_d_items_fk; +ALTER TABLE mimiciv_icu.inputevents +ADD CONSTRAINT inputevents_d_items_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_icu.d_items (itemid); + +-- outputevents + +ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_patients_fk; +ALTER TABLE mimiciv_icu.outputevents +ADD CONSTRAINT outputevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_admissions_fk; +ALTER TABLE mimiciv_icu.outputevents +ADD CONSTRAINT outputevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_icustays_fk; +ALTER TABLE mimiciv_icu.outputevents +ADD CONSTRAINT outputevents_icustays_fk + FOREIGN KEY (stay_id) + REFERENCES mimiciv_icu.icustays (stay_id); + +ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_d_items_fk; +ALTER TABLE mimiciv_icu.outputevents +ADD CONSTRAINT outputevents_d_items_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_icu.d_items (itemid); + +-- procedureevents + +ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_patients_fk; +ALTER TABLE mimiciv_icu.procedureevents +ADD CONSTRAINT procedureevents_patients_fk + FOREIGN KEY (subject_id) + REFERENCES mimiciv_hosp.patients (subject_id); + +ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_admissions_fk; +ALTER TABLE mimiciv_icu.procedureevents +ADD CONSTRAINT procedureevents_admissions_fk + FOREIGN KEY (hadm_id) + REFERENCES mimiciv_hosp.admissions (hadm_id); + +ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_icustays_fk; +ALTER TABLE mimiciv_icu.procedureevents +ADD CONSTRAINT procedureevents_icustays_fk + FOREIGN KEY (stay_id) + REFERENCES mimiciv_icu.icustays (stay_id); + +ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_d_items_fk; +ALTER TABLE mimiciv_icu.procedureevents +ADD CONSTRAINT procedureevents_d_items_fk + FOREIGN KEY (itemid) + REFERENCES mimiciv_icu.d_items (itemid); diff --git a/mimic-iv/buildmimic/postgres/docker/src/04_index.sql b/mimic-iv/buildmimic/postgres/docker/src/04_index.sql new file mode 100644 index 000000000..948a0d4c0 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/src/04_index.sql @@ -0,0 +1,232 @@ +-------------------------------------- +-------------------------------------- +-- Indexes for all MIMIC-IV modules -- +-------------------------------------- +-------------------------------------- + +---------- +-- hosp -- +---------- + +SET search_path TO mimiciv_hosp; + +-- admissions + +DROP INDEX IF EXISTS admissions_idx01; +CREATE INDEX admissions_idx01 + ON admissions (admittime, dischtime, deathtime); + +-- d_icd_diagnoses + +DROP INDEX IF EXISTS D_ICD_DIAG_idx02; +CREATE INDEX D_ICD_DIAG_idx02 + ON D_ICD_DIAGNOSES (LONG_TITLE); + +-- D_ICD_PROCEDURES + +DROP INDEX IF EXISTS D_ICD_PROC_idx02; +CREATE INDEX D_ICD_PROC_idx02 + ON D_ICD_PROCEDURES (LONG_TITLE); + +-- drgcodes + +DROP INDEX IF EXISTS drgcodes_idx01; +CREATE INDEX drgcodes_idx01 + ON drgcodes (drg_code, drg_type); + +DROP INDEX IF EXISTS drgcodes_idx02; +CREATE INDEX drgcodes_idx02 + ON drgcodes (description, drg_severity); + +-- d_labitems + +DROP INDEX IF EXISTS d_labitems_idx01; +CREATE INDEX d_labitems_idx01 + ON d_labitems (label, fluid, category); + +-- emar_detail + +DROP INDEX IF EXISTS emar_detail_idx01; +CREATE INDEX emar_detail_idx01 + ON emar_detail (pharmacy_id); + +DROP INDEX IF EXISTS emar_detail_idx02; +CREATE INDEX emar_detail_idx02 + ON emar_detail (product_code); + +DROP INDEX IF EXISTS emar_detail_idx03; +CREATE INDEX emar_detail_idx03 + ON emar_detail (route, site, side); + +DROP INDEX IF EXISTS EMAR_DET_idx04; +CREATE INDEX EMAR_DET_idx04 + ON EMAR_DETAIL (PRODUCT_DESCRIPTION); + +-- emar + +DROP INDEX IF EXISTS emar_idx01; +CREATE INDEX emar_idx01 + ON emar (poe_id); + +DROP INDEX IF EXISTS emar_idx02; +CREATE INDEX emar_idx02 + ON emar (pharmacy_id); + +DROP INDEX IF EXISTS emar_idx03; +CREATE INDEX emar_idx03 + ON emar (charttime, scheduletime, storetime); + +DROP INDEX IF EXISTS emar_idx04; +CREATE INDEX emar_idx04 + ON emar (medication); + +-- HCPCSEVENTS + +DROP INDEX IF EXISTS HCPCSEVENTS_idx04; +CREATE INDEX HCPCSEVENTS_idx04 + ON HCPCSEVENTS (SHORT_DESCRIPTION); + +-- labevents + +DROP INDEX IF EXISTS labevents_idx01; +CREATE INDEX labevents_idx01 + ON labevents (charttime, storetime); + +DROP INDEX IF EXISTS labevents_idx02; +CREATE INDEX labevents_idx02 + ON labevents (specimen_id); + +-- microbiologyevents + +DROP INDEX IF EXISTS microbiologyevents_idx01; +CREATE INDEX microbiologyevents_idx01 + ON microbiologyevents (chartdate, charttime, storedate, storetime); + +DROP INDEX IF EXISTS microbiologyevents_idx02; +CREATE INDEX microbiologyevents_idx02 + ON microbiologyevents (spec_itemid, test_itemid, org_itemid, ab_itemid); + +DROP INDEX IF EXISTS microbiologyevents_idx03; +CREATE INDEX microbiologyevents_idx03 + ON microbiologyevents (micro_specimen_id); + +-- patients +DROP INDEX IF EXISTS patients_idx01; +CREATE INDEX patients_idx01 + ON patients (anchor_age); + +DROP INDEX IF EXISTS patients_idx02; +CREATE INDEX patients_idx02 + ON patients (anchor_year); + +-- pharmacy + +DROP INDEX IF EXISTS pharmacy_idx01; +CREATE INDEX pharmacy_idx01 + ON pharmacy (poe_id); + +DROP INDEX IF EXISTS pharmacy_idx02; +CREATE INDEX pharmacy_idx02 + ON pharmacy (starttime, stoptime); + +DROP INDEX IF EXISTS pharmacy_idx03; +CREATE INDEX pharmacy_idx03 + ON pharmacy (medication); + +DROP INDEX IF EXISTS pharmacy_idx04; +CREATE INDEX pharmacy_idx04 + ON pharmacy (route); + +-- poe + +DROP INDEX IF EXISTS poe_idx01; +CREATE INDEX poe_idx01 + ON poe (order_type); + +-- prescriptions + +DROP INDEX IF EXISTS prescriptions_idx01; +CREATE INDEX prescriptions_idx01 + ON prescriptions (starttime, stoptime); + +-- transfers + +DROP INDEX IF EXISTS transfers_idx01; +CREATE INDEX transfers_idx01 + ON transfers (hadm_id); + +DROP INDEX IF EXISTS transfers_idx02; +CREATE INDEX transfers_idx02 + ON transfers (intime); + +DROP INDEX IF EXISTS transfers_idx03; +CREATE INDEX transfers_idx03 + ON transfers (careunit); + +--------- +-- icu -- +--------- + +SET search_path TO mimiciv_icu; + +-- chartevents + +DROP INDEX IF EXISTS chartevents_idx01; +CREATE INDEX chartevents_idx01 + ON chartevents (charttime, storetime); + +-- datetimeevents + +DROP INDEX IF EXISTS datetimeevents_idx01; +CREATE INDEX datetimeevents_idx01 + ON datetimeevents (charttime, storetime); + +DROP INDEX IF EXISTS datetimeevents_idx02; +CREATE INDEX datetimeevents_idx02 + ON datetimeevents (value); + +-- d_items + +DROP INDEX IF EXISTS d_items_idx01; +CREATE INDEX d_items_idx01 + ON d_items (label, abbreviation); + +DROP INDEX IF EXISTS d_items_idx02; +CREATE INDEX d_items_idx02 + ON d_items (category); + +-- icustays + +DROP INDEX IF EXISTS icustays_idx01; +CREATE INDEX icustays_idx01 + ON icustays (first_careunit, last_careunit); + +DROP INDEX IF EXISTS icustays_idx02; +CREATE INDEX icustays_idx02 + ON icustays (intime, outtime); + +-- inputevents + +DROP INDEX IF EXISTS inputevents_idx01; +CREATE INDEX inputevents_idx01 + ON inputevents (starttime, endtime); + +DROP INDEX IF EXISTS inputevents_idx02; +CREATE INDEX inputevents_idx02 + ON inputevents (ordercategorydescription); + +-- outputevents + +DROP INDEX IF EXISTS outputevents_idx01; +CREATE INDEX outputevents_idx01 + ON outputevents (charttime, storetime); + +-- procedureevents + +DROP INDEX IF EXISTS procedureevents_idx01; +CREATE INDEX procedureevents_idx01 + ON procedureevents (starttime, endtime); + +DROP INDEX IF EXISTS procedureevents_idx02; +CREATE INDEX procedureevents_idx02 + ON procedureevents (ordercategoryname); From 56e0b9b14dbb70d6d18cdad337d52104e8738375 Mon Sep 17 00:00:00 2001 From: mlbernauer Date: Fri, 14 Jun 2024 22:19:15 -0600 Subject: [PATCH 2/8] update --- mimic-iv/buildmimic/postgres/docker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml index b1e9c4ced..f0b459767 100644 --- a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -24,7 +24,7 @@ services: # (hint: check ls -l ./mimic-data/) this should be the user of host # machine. this will also be the user you will use to authenticate with # postgres - POSTGRES_USER: mlbernauer + POSTGRES_USER: # this will be the password for authenticating with postgres...set to # whatever you want - POSTGRES_PASSWORD: pass123 + POSTGRES_PASSWORD: From 481126b63fb23c439ee7d32f3130d8ead1cb21b2 Mon Sep 17 00:00:00 2001 From: mlbernauer Date: Wed, 19 Jun 2024 09:08:38 -0600 Subject: [PATCH 3/8] made db name consistent with concepts scripts --- mimic-iv/buildmimic/postgres/docker/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml index f0b459767..418689ce6 100644 --- a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -16,15 +16,15 @@ services: # this maps a directory on the host machine (i.e. ./mimic-db/) to the # container so that the data in the database can persist after container # has been shut down. - - ./mimic-db/:/var/lib/postgresql/data/ + - /data/mimic-iv-db/:/var/lib/postgresql/data/ environment: # this will be the name of the database within Postgres. - POSTGRES_DB: mimiciv + POSTGRES_DB: physionet-data # set this to the same user that owns the directory storing the .csv data # (hint: check ls -l ./mimic-data/) this should be the user of host # machine. this will also be the user you will use to authenticate with # postgres - POSTGRES_USER: + POSTGRES_USER: mlbernauer # this will be the password for authenticating with postgres...set to # whatever you want - POSTGRES_PASSWORD: + POSTGRES_PASSWORD: pass123 From bb84429a6e12455f498029a1dea49b95ce63969c Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 10 Aug 2026 19:28:51 -0400 Subject: [PATCH 4/8] rework docker and add duckdb docker build --- .github/actions/load-mimic-duckdb/action.yml | 12 +- .github/actions/setup-duckdb/action.yml | 3 +- .gitignore | 7 +- mimic-iv/buildmimic/download_data.sh | 40 ++ mimic-iv/buildmimic/duckdb/README.md | 48 +- mimic-iv/buildmimic/duckdb/build_mimic.sh | 216 +++++++ .../buildmimic/duckdb/docker/.env.example | 20 + mimic-iv/buildmimic/duckdb/docker/Dockerfile | 40 ++ .../duckdb/docker/Dockerfile.dockerignore | 10 + mimic-iv/buildmimic/duckdb/docker/README.md | 107 ++++ .../duckdb/docker/docker-compose.yml | 22 + mimic-iv/buildmimic/duckdb/filter_db.sql | 45 ++ mimic-iv/buildmimic/duckdb/import_duckdb.sh | 132 ----- mimic-iv/buildmimic/postgres/README.md | 31 + mimic-iv/buildmimic/postgres/build_mimic.sh | 194 ++++++ .../buildmimic/postgres/docker/.env.example | 24 + .../buildmimic/postgres/docker/Dockerfile | 22 +- .../postgres/docker/Dockerfile.dockerignore | 6 + mimic-iv/buildmimic/postgres/docker/README.md | 116 +++- .../postgres/docker/docker-compose.yml | 71 ++- .../postgres/docker/download_data.sh | 4 - .../postgres/docker/src/01_create.sql | 519 ----------------- .../postgres/docker/src/02_load_gz.sql | 37 -- .../postgres/docker/src/03_constrant.sql | 551 ------------------ .../postgres/docker/src/04_index.sql | 232 -------- 25 files changed, 953 insertions(+), 1556 deletions(-) create mode 100755 mimic-iv/buildmimic/download_data.sh create mode 100755 mimic-iv/buildmimic/duckdb/build_mimic.sh create mode 100644 mimic-iv/buildmimic/duckdb/docker/.env.example create mode 100644 mimic-iv/buildmimic/duckdb/docker/Dockerfile create mode 100644 mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore create mode 100644 mimic-iv/buildmimic/duckdb/docker/README.md create mode 100644 mimic-iv/buildmimic/duckdb/docker/docker-compose.yml create mode 100644 mimic-iv/buildmimic/duckdb/filter_db.sql delete mode 100755 mimic-iv/buildmimic/duckdb/import_duckdb.sh create mode 100755 mimic-iv/buildmimic/postgres/build_mimic.sh create mode 100644 mimic-iv/buildmimic/postgres/docker/.env.example create mode 100644 mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore delete mode 100755 mimic-iv/buildmimic/postgres/docker/download_data.sh delete mode 100644 mimic-iv/buildmimic/postgres/docker/src/01_create.sql delete mode 100644 mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql delete mode 100644 mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql delete mode 100644 mimic-iv/buildmimic/postgres/docker/src/04_index.sql diff --git a/.github/actions/load-mimic-duckdb/action.yml b/.github/actions/load-mimic-duckdb/action.yml index b8d92f935..e201d1901 100644 --- a/.github/actions/load-mimic-duckdb/action.yml +++ b/.github/actions/load-mimic-duckdb/action.yml @@ -1,5 +1,5 @@ name: "Load MIMIC-IV demo into DuckDB" -description: "Builds a DuckDB database from the demo hosp/icu data using import_duckdb.sh. Requires the duckdb CLI on PATH." +description: "Builds a DuckDB database from the demo hosp/icu data using build_mimic.sh. Requires the duckdb CLI on PATH." inputs: mimic_data_dir: @@ -18,11 +18,17 @@ runs: shell: bash # Resolve paths at shell time so $GITHUB_WORKSPACE points at the # container-mounted path. - # `echo n` answers the overwrite prompt defensively. + # + # Load only. The concepts are transpiled and built by concepts-duckdb.yml + # after this action runs, and the row counts are checked by the calling + # workflow, so build_mimic.sh must not do either here. + env: + MIMIC_MAKE_CONCEPTS: "false" + MIMIC_VALIDATE: "false" run: | data_dir="${{ inputs.mimic_data_dir }}" data_dir="${data_dir:-$GITHUB_WORKSPACE}" db_file="${{ inputs.db_file }}" db_file="${db_file:-$GITHUB_WORKSPACE/mimic4.db}" cd "$GITHUB_WORKSPACE/mimic-iv/buildmimic/duckdb" - echo n | ./import_duckdb.sh "$data_dir" "$db_file" + ./build_mimic.sh "$data_dir" "$db_file" diff --git a/.github/actions/setup-duckdb/action.yml b/.github/actions/setup-duckdb/action.yml index 91ba9350a..35e3015e5 100644 --- a/.github/actions/setup-duckdb/action.yml +++ b/.github/actions/setup-duckdb/action.yml @@ -5,7 +5,8 @@ inputs: version: description: "DuckDB release version to install." required: false - default: "1.1.3" + # Keep in step with DUCKDB_VERSION in mimic-iv/buildmimic/duckdb/docker. + default: "1.4.5" runs: using: "composite" diff --git a/.gitignore b/.gitignore index d0f33f313..661750085 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -## Local docker-postgres raw data (do not commit large CSVs) -mimic-iv/docker-postgres/mimic-data/ +## Raw data downloaded for local builds (do not commit large CSVs) +mimic-data/ ## Allow example env, ignore real env -mimic-iv/docker-postgres/.env +.env +!.env.example # duckdb / sqlite db files *.db diff --git a/mimic-iv/buildmimic/download_data.sh b/mimic-iv/buildmimic/download_data.sh new file mode 100755 index 000000000..0131a6999 --- /dev/null +++ b/mimic-iv/buildmimic/download_data.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Download MIMIC-IV from PhysioNet into a directory any of the builds can use. +# +# Usage: ./download_data.sh [destination] [physionet-username] +# +# destination where to put the data (default ./mimic-data) +# physionet-username also read from $PHYSIONET_USER, otherwise prompted for +# +# Requires a PhysioNet account credentialed for MIMIC-IV. The password is always +# requested interactively, so it never reaches the process list or shell history. +# +# Shared by the postgres and duckdb builds: both want the same hosp/ and icu/ +# layout, so there is one copy of this here rather than one per engine. +# +# This is a convenience only. If you already have the data, point MIMIC_DATA_DIR +# at it instead; any directory with hosp/ and icu/ subfolders will do. +set -euo pipefail + +# The build scripts target the current release of MIMIC-IV. +readonly MIMIC_VERSION="3.1" + +DEST="${1:-./mimic-data}" +USERNAME="${2:-${PHYSIONET_USER:-}}" + +if [ -z "${USERNAME}" ]; then + read -rp "PhysioNet username: " USERNAME +fi + +echo "Downloading MIMIC-IV v${MIMIC_VERSION} to ${DEST}" + +# -nH --cut-dirs=4 strips physionet.org/files/mimiciv// from the paths, +# leaving the hosp/ and icu/ subfolders directly under ${DEST}. That is the +# layout buildmimic/postgres/load_gz.sql expects. +wget -r -N -c -np -nH --cut-dirs=4 \ + -A '*.csv.gz' \ + -P "${DEST}" \ + --user "${USERNAME}" --ask-password \ + "https://physionet.org/files/mimiciv/${MIMIC_VERSION}/" + +echo "Downloaded $(find "${DEST}" -name '*.csv.gz' | wc -l | tr -d ' ') files to ${DEST}" diff --git a/mimic-iv/buildmimic/duckdb/README.md b/mimic-iv/buildmimic/duckdb/README.md index 60bbd3d52..28a4f728b 100644 --- a/mimic-iv/buildmimic/duckdb/README.md +++ b/mimic-iv/buildmimic/duckdb/README.md @@ -24,12 +24,15 @@ which you can obtain by either installing [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) or [Cygwin](https://www.cygwin.com/). +If you would rather not install DuckDB at all, the [docker](docker) folder +builds the same database in a container with a pinned DuckDB version. + ## Set-up ### Quick overview 1. [Install](https://duckdb.org/docs/installation/) the CLI version of DuckDB -2. [Download](https://physionet.org/content/mimiciv/2.0) the MIMIC-IV files +2. [Download](https://physionet.org/content/mimiciv/) the MIMIC-IV files 3. Create DuckDB database and load data ### Install DuckDB @@ -41,6 +44,9 @@ the CLI version of DuckDB. You will need to place the `duckdb` binary in a folder on your environment path, e.g. `/usr/local/bin`. +These scripts are built and tested against the 1.4.x LTS line (currently +1.4.5), which is what CI uses. + ### Download MIMIC-IV files Download the CSV files for [MIMIC-IV](https://physionet.org/content/mimiciv/) @@ -63,43 +69,38 @@ mimic_data_dir The CSV files can be uncompressed (end in `.csv`) or compressed (end in `.csv.gz`). -The easiest way to download them is to open a terminal then run: +The easiest way to download them is to use the shared download script, which +wraps `wget` and puts the files in the layout the build expects: -``` -wget -r -N -c -np --user YOURUSERNAME --ask-password https://physionet.org/files/mimiciv/2.2/ +```sh +../download_data.sh ./mimic-data YOURUSERNAME ``` -Replace `YOURUSERNAME` with your physionet username. - -This will make you `mimic_data_dir` be `physionet.org/files/mimiciv/2.2`. +Replace `YOURUSERNAME` with your physionet username; you will be prompted for +the password. This makes your `mimic_data_dir` be `./mimic-data`. # Create DuckDB database and load data The last step requires creating a DuckDB database and loading the data into it. -You can do all of this with one shell script, `import_duckdb.sh`, +You can do all of this with one shell script, `build_mimic.sh`, located in this repository. -See the help for it below: - ```sh -$ ./import_duckdb.sh -h -./import_duckdb.sh: -USAGE: ./import_duckdb.sh mimic_data_dir [output_db] -WHERE: - mimic_data_dir directory that contains csv.gz or csv files - output_db: optional filename for duckdb file (default: mimic4.db) -$ +$ ./build_mimic.sh -h +Usage: build_mimic.sh [output_db] + mimic_data_dir the directory containing the hosp/ and icu/ subfolders + output_db filename for the duckdb file (default mimic4.db) ``` Here's an example invocation that will make the database in the default "mimic4.db": ```sh -$ ./import_duckdb.sh physionet.org/files/mimiciv/2.2 +$ ./build_mimic.sh ./mimic-data <... output of script snipped ...> -Successfully finished loading data into mimic4.db. +MIMIC-IV build complete: /path/to/mimic4.db $ ls -lh mimic4.db -rw-rw-r--. 1 myuser mygroup 93G May 26 16:11 mimic4.db @@ -109,6 +110,15 @@ The script will print out progress as it goes. Be patient, this can take minutes to hours to load depending on your computer's configuration. +Beyond loading the data, it also derives the concepts from +[concepts_duckdb](../../concepts_duckdb) into the `mimiciv_derived` schema and +checks the loaded tables against known row counts. Set `MIMIC_MAKE_CONCEPTS` or +`MIMIC_VALIDATE` to `false` to skip either. + +Each step is recorded in a `mimiciv_build_progress` table inside the database +file, and tables that already hold rows are skipped, so re-running the script +after an interruption resumes rather than starting over. + * It took 16m25s on a Fedora 34 workstation with duckdb v 0.2.6, a btrfs filesystem with ztsd level 1 compression, AMD Ryzen 3900X, 32 GB RAM, Samsung 970 Evo NVMe SSD. * It took ~10m on a Mac M1 Max 2021, 32 GB RAM. diff --git a/mimic-iv/buildmimic/duckdb/build_mimic.sh b/mimic-iv/buildmimic/duckdb/build_mimic.sh new file mode 100755 index 000000000..048819988 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/build_mimic.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Build MIMIC-IV in DuckDB: create the schema, load the data, then derive the +# concepts. +# +# Usage: +# ./build_mimic.sh [output_db] +# +# is the directory holding the hosp/ and icu/ subfolders. +# Compressed (.csv.gz) and uncompressed (.csv) data are both supported; DuckDB +# decompresses by file extension, so no separate loader is needed. +# +# Each step is recorded in a progress table once it completes, so an +# interrupted build will resume where it stopped. The table lives inside the +# database file, so the resume state travels with it. +# +# Environment: +# MIMIC_DATA_DIR used when is not given +# MIMIC_DB used when [output_db] is not given (default mimic4.db) +# MIMIC_MAKE_CONCEPTS derive the concepts into mimiciv_derived (default true) +# MIMIC_VALIDATE check tables against expected row counts (default true) +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +readonly SCRIPT_DIR +readonly CONCEPTS_DIR="${SCRIPT_DIR}/../../concepts_duckdb" +# The schema and the row count checks are shared with the postgres build rather +# than duplicated. create.sql needs the patches in patch_create_sql below; +# validate.sql is plain ANSI SQL and runs unmodified. +readonly POSTGRES_DIR="${SCRIPT_DIR}/../postgres" +readonly PROGRESS_TABLE=mimiciv_build_progress + +DATA_DIR="" +DB="" + +# -f tells DuckDB to stop at the first failing statement and exit non-zero. +duckdb_run() { duckdb "${DB}" -f "$1"; } +duckdb_cmd() { duckdb "${DB}" -c "$1"; } +duckdb_val() { duckdb "${DB}" -noheader -list -c "$1"; } + +# Lowercase string before checking value +is_true() { [ "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" = "true" ]; } + +usage() { + echo "Usage: $(basename "$0") [output_db]" >&2 + echo " mimic_data_dir the directory containing the hosp/ and icu/ subfolders" >&2 + echo " output_db filename for the duckdb file (default mimic4.db)" >&2 +} + +step_done() { + [ "$(duckdb_val "SELECT EXISTS (SELECT 1 FROM ${PROGRESS_TABLE} WHERE step = '$1')")" = "true" ] +} + +run_step() { + local step=$1 + shift + if step_done "${step}"; then + echo "== ${step}: already done, skipping" + return + fi + echo "== ${step}: starting" + "$@" + duckdb_cmd "INSERT INTO ${PROGRESS_TABLE} (step) VALUES ('${step}') ON CONFLICT DO NOTHING" >/dev/null + echo "== ${step}: done" +} + +# The schema is defined once, in the postgres build. Three things in it are not +# valid or not desirable in DuckDB: +# 1. TIMESTAMP(NN) -- DuckDB does not accept a precision argument. +# 2. NOT NULL on mimiciv_hosp.microbiologyevents.spec_type_desc -- there is one +# (!) zero-length string, which the import treats as NULL. +# 3. NOT NULL on mimiciv_hosp.prescriptions.drug -- likewise, zero-length +# strings treated as NULL. +patch_create_sql() { + sed -E \ + -e 's/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' \ + -e 's/spec_type_desc(.+)NOT NULL/spec_type_desc\1/g' \ + -e 's/drug +(VARCHAR.+)NOT NULL/drug \1/g' \ + "${POSTGRES_DIR}/create.sql" +} + +create_schema() { + patch_create_sql | duckdb "${DB}" +} + +# hosp/admissions.csv.gz -> mimiciv_hosp.admissions +make_table_name() { + local path=$1 basename dirname + basename=${path##*/} + dirname=${path%/*} + dirname=${dirname##*/} + printf 'mimiciv_%s.%s' "${dirname}" "${basename%%.*}" +} + +# A COPY is a single transaction, so a table is either fully loaded or empty. +# Even a single row indicates completion of the data load. +table_has_rows() { + [ "$(duckdb_val "SELECT EXISTS (SELECT 1 FROM $1 LIMIT 1)")" = "true" ] +} + +load_data() { + local file table + while IFS= read -r file; do + table=$(make_table_name "${file}") + + # A file with no matching table in create.sql is reported and skipped + # rather than failing the whole build. + if ! duckdb_val "SELECT 1 FROM ${table} LIMIT 0" >/dev/null 2>&1; then + echo " ${table}: not in the schema, skipping ${file##*/}" + continue + fi + if table_has_rows "${table}"; then + echo " ${table}: already loaded, skipping" + continue + fi + echo " ${table}: loading" + duckdb_cmd "COPY ${table} FROM '${file}' (HEADER, DELIM ',', QUOTE '\"', ESCAPE '\"')" >/dev/null + done < <(find "${DATA_DIR}"/hosp "${DATA_DIR}"/icu -type f \ + \( -name '*.csv' -o -name '*.csv.gz' \) | sort) +} + +make_concepts() { + # duckdb.sql pulls in the individual concepts with .read, which resolves + # relative to the working directory rather than to the script. + cd "${CONCEPTS_DIR}" + duckdb "${DB}" -f duckdb.sql +} + +validate() { + # default validate the full DB + local script=validate.sql + + # The demo is a 100 patient subset of MIMIC-IV, so it has its own set of + # expected row counts. + if [ "$(duckdb_val 'SELECT count(*) FROM mimiciv_hosp.patients')" -eq 100 ]; then + echo " 100 patients found, validating against the MIMIC-IV demo counts" + script=validate_demo.sql + fi + + local output + output=$(duckdb "${DB}" -f "${POSTGRES_DIR}/${script}") + echo "${output}" + + # Mismatches indicate (1) failure to load (zero rows) or (2) mismatch mimic version + if echo "${output}" | grep -q FAILED; then + echo "WARNING: some tables do not have the expected number of rows." >&2 + fi +} + +resolve_data_dir() { + local dir=${1:-${MIMIC_DATA_DIR:-}} + + if [ -z "${dir}" ]; then + usage + exit 1 + fi + if [ ! -d "${dir}/hosp" ] || [ ! -d "${dir}/icu" ]; then + echo "ERROR: ${dir} must contain the hosp/ and icu/ subfolders of MIMIC-IV." >&2 + exit 1 + fi + if [ -z "$(find "${dir}/hosp" -maxdepth 1 \( -name '*.csv' -o -name '*.csv.gz' \) -print -quit)" ]; then + echo "ERROR: no .csv or .csv.gz files found in ${dir}/hosp." >&2 + exit 1 + fi + + DATA_DIR=$(cd -- "${dir}" && pwd) +} + +# make_concepts has to cd into the concepts directory, so a relative path here +# would resolve against the wrong directory and silently build a second, +# empty database. +resolve_db() { + local db=${1:-${MIMIC_DB:-mimic4.db}} + local dir=${db%/*} + + [ "${dir}" = "${db}" ] && dir=. + if [ ! -d "${dir}" ]; then + echo "ERROR: ${dir} does not exist, cannot create ${db} in it." >&2 + exit 1 + fi + + DB="$(cd -- "${dir}" && pwd)/${db##*/}" +} + +main() { + case "${1:-}" in + -h|--help) usage; exit 0 ;; + esac + + resolve_data_dir "${1:-}" + resolve_db "${2:-}" + echo "Loading from ${DATA_DIR} into ${DB} using DuckDB $(duckdb --version)" + + duckdb_cmd "CREATE TABLE IF NOT EXISTS ${PROGRESS_TABLE} ( + step text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() + )" >/dev/null + + run_step create create_schema + run_step load load_data + + if is_true "${MIMIC_MAKE_CONCEPTS:-true}"; then + run_step concepts make_concepts + else + echo "== concepts: MIMIC_MAKE_CONCEPTS is not 'true', skipping" + fi + + if is_true "${MIMIC_VALIDATE:-true}"; then + run_step validate validate + else + echo "== validate: MIMIC_VALIDATE is not 'true', skipping" + fi + + echo "MIMIC-IV build complete: ${DB}" +} + +main "$@" diff --git a/mimic-iv/buildmimic/duckdb/docker/.env.example b/mimic-iv/buildmimic/duckdb/docker/.env.example new file mode 100644 index 000000000..f2b7a9b0d --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/.env.example @@ -0,0 +1,20 @@ +# Copy to .env and edit as needed: cp .env.example .env + +# DuckDB version used to build the database. 1.4.x is the current LTS. +DUCKDB_VERSION=1.4.5 + +# Directory holding the MIMIC-IV .csv.gz files, containing hosp/ and icu/ +# subfolders. Populate it by running ../../download_data.sh +MIMIC_DATA_DIR=./mimic-data + +# Where the finished database is written on the host, and what to call it. +MIMIC_OUTPUT_DIR=./mimic-db +MIMIC_DB_NAME=mimic4.db + +# Derive the concepts from mimic-iv/concepts_duckdb once the data is loaded. +# Set to false to load the raw tables only. +MIMIC_MAKE_CONCEPTS=true + +# Check the loaded tables against the expected row counts. The demo and the full +# dataset are detected automatically. +MIMIC_VALIDATE=true diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile b/mimic-iv/buildmimic/duckdb/docker/Dockerfile new file mode 100644 index 000000000..ded652d0a --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile @@ -0,0 +1,40 @@ +# Image for the job that builds MIMIC-IV into a DuckDB file. +# Writes a .db file into a mounted directory and exits. +# +# The build context is the mimic-iv/ directory, and the repository layout is +# kept intact inside the image so that build_mimic.sh finds the shared postgres +# schema beside it and the concepts two directories up. +# +# There is no official DuckDB image, so the CLI is fetched from the pinned +# GitHub release. DUCKDB_VERSION decides which one; see docker-compose.yml. +ARG DUCKDB_VERSION=1.4.5 + +FROM debian:stable-slim + +# Repeated because the ARG above the first FROM is outside the build stage. +ARG DUCKDB_VERSION +# Set automatically by buildkit: amd64 or arm64. DuckDB publishes both under +# exactly these names. +ARG TARGETARCH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl unzip \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL -o /tmp/duckdb.zip \ + "https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-${TARGETARCH}.zip" \ + && unzip -q /tmp/duckdb.zip -d /usr/local/bin \ + && rm /tmp/duckdb.zip \ + && chmod +x /usr/local/bin/duckdb \ + && duckdb --version + +COPY buildmimic/duckdb/build_mimic.sh /mimic/buildmimic/duckdb/ +# create.sql, validate.sql and validate_demo.sql are shared with the postgres +# build; build_mimic.sh reads them from ../postgres. +COPY buildmimic/postgres/create.sql buildmimic/postgres/validate*.sql /mimic/buildmimic/postgres/ +COPY concepts_duckdb/ /mimic/concepts_duckdb/ + +# The database is written here. +WORKDIR /out + +ENTRYPOINT ["/mimic/buildmimic/duckdb/build_mimic.sh"] +CMD ["/data"] diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..badd2c070 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +# The build context is mimic-iv/ +# Exclude everything, then add back only what the image needs. +* +!buildmimic/duckdb/build_mimic.sh +!buildmimic/postgres/create.sql +!buildmimic/postgres/validate.sql +!buildmimic/postgres/validate_demo.sql +!concepts_duckdb/ +# Explicitly avoid including output db files +*.db diff --git a/mimic-iv/buildmimic/duckdb/docker/README.md b/mimic-iv/buildmimic/duckdb/docker/README.md new file mode 100644 index 000000000..6ba50bd62 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/README.md @@ -0,0 +1,107 @@ +# MIMIC-IV with DuckDB and Docker + +Build a MIMIC-IV DuckDB database in a container. The build scripts live in the +parent folder. It creates the schema, loads the data, (optionally, default true) +derives the concepts from [concepts_duckdb](../../../concepts_duckdb), and +validates the result. The container is only used to reproducibly build the +database: it is not necessary for analyzing it afterward. + +## Requirements + +* Docker, with Compose v2. +* The dataset (`../../download_data.sh` is a convenience for downloading data from PhysioNet). +* Disk space. The compressed download is about 10 GB; the resulting `.db` file + is roughly 25 GB, plus a few GB if you build the concepts. The + database is written to a bind mount on your host, so this space comes from + your disk rather than the Docker Desktop virtual disk, but the *load* still + needs Docker to have room to work. + +## Quickstart + +```bash +# 1. Configure. Set DUCKDB_VERSION to match your local duckdb. +# Make sure MIMIC_DATA_DIR has the hosp/ and icu/ subfolders. +cp .env.example .env + +# 2. Download the data, if you do not already have it (about 10 GB). +../../download_data.sh + +# 3. Build. Runs in the foreground so you can watch it; add -d to detach. +docker compose up +``` + +## Notes + +### DuckDB version + +The container's DuckDB and the `duckdb` on your machine are two separate installs. +The container has 1.4.5 which is the LTS version. `duckdb` is mostly backward +compatible but be aware of differences. + +Check what you have: + +```bash +duckdb --version +``` + +and set `DUCKDB_VERSION` in `.env` to match before building. If you later see an +error about the storage version when opening the file, the fix is to upgrade +your local DuckDB or rebuild with a lower `DUCKDB_VERSION`. + +The result is `./mimic-db/mimic4.db`. Change `MIMIC_OUTPUT_DIR` and +`MIMIC_DB_NAME` in `.env` to put it somewhere else. + +Loading takes on the order of an hour for the full dataset on a reasonable +machine, plus time for the concepts. Watch it with: + +```bash +docker compose logs -f mimic-build +``` + +### If the build is interrupted + +Run `docker compose up` again. The build records each step as it completes and +skips tables that are already populated, so it resumes rather than starting +over. Progress is tracked in the `mimiciv_build_progress` table inside the +database file. + +```bash +duckdb ./mimic-db/mimic4.db -c 'TABLE mimiciv_build_progress' +``` + +To discard everything and start clean, delete the file: + +```bash +rm ./mimic-db/mimic4.db +docker compose up --build +``` + +Use `--build` whenever you change the SQL or `DUCKDB_VERSION`, otherwise +Compose reuses the existing image. + +### Settings + +All are set in `.env`; see `.env.example` for the defaults. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `DUCKDB_VERSION` | `1.4.5` | DuckDB version to build with. Match your local `duckdb`. | +| `MIMIC_DATA_DIR` | `./mimic-data` | Directory holding `hosp/` and `icu/`. | +| `MIMIC_OUTPUT_DIR` | `./mimic-db` | Host directory the database is written to. | +| `MIMIC_DB_NAME` | `mimic4.db` | Filename of the database. | +| `MIMIC_MAKE_CONCEPTS` | `true` | Derive the concepts into `mimiciv_derived`. | +| `MIMIC_VALIDATE` | `true` | Check the loaded tables against expected row counts. | + +## Using the database + +```bash +# from the host, with your own duckdb +duckdb ./mimic-db/mimic4.db + +# otherwise, from inside a throwaway container using the pinned version (not recommended) +docker compose run --rm --entrypoint duckdb mimic-build /out/mimic4.db +``` + +On Linux the file is written as `root`, since that is what the container runs +as. `sudo chown "$USER" ./mimic-db/mimic4.db` if you would rather own it. Docker +Desktop on macOS and Windows maps ownership to you automatically. diff --git a/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml new file mode 100644 index 000000000..6dec5c7ba --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml @@ -0,0 +1,22 @@ +services: + # Loads the data and derives the concepts, then exits. + # Re-running it resumes an interrupted build. + # + # As DuckDB is embedded, this container's only purpose is to + # reproducibly create the database file. + mimic-build: + build: + # mimic-iv/, so the build can reach buildmimic/ and concepts_duckdb + context: ../../.. + dockerfile: buildmimic/duckdb/docker/Dockerfile + args: + DUCKDB_VERSION: ${DUCKDB_VERSION:-1.4.5} + environment: + MIMIC_DB: /out/${MIMIC_DB_NAME:-mimic4.db} + MIMIC_MAKE_CONCEPTS: ${MIMIC_MAKE_CONCEPTS:-true} + MIMIC_VALIDATE: ${MIMIC_VALIDATE:-true} + volumes: + # The .csv.gz files, read-only. Must contain hosp/ and icu/ subfolders. + - ${MIMIC_DATA_DIR:-./mimic-data}:/data:ro + # The finished database is output to a bind mount. + - ${MIMIC_OUTPUT_DIR:-./mimic-db}:/out diff --git a/mimic-iv/buildmimic/duckdb/filter_db.sql b/mimic-iv/buildmimic/duckdb/filter_db.sql new file mode 100644 index 000000000..c152c366f --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/filter_db.sql @@ -0,0 +1,45 @@ +-- remove all but the first 10 subjects +-- mimiciv_hosp.demo_subject_id +DELETE FROM mimiciv_hosp.patients +WHERE subject_id NOT IN +(SELECT subject_id FROM mimiciv_hosp.patients ORDER BY subject_id LIMIT 10); + +-- apply this to all the other tables +DELETE FROM mimiciv_hosp.admissions WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.diagnoses_icd WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.drgcodes WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.emar WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.emar_detail WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.hcpcsevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.labevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.microbiologyevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.omr WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.pharmacy WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.poe WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.poe_detail WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.prescriptions WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.procedures_icd WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.services WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_hosp.transfers WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); + +DELETE FROM mimiciv_icu.chartevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.datetimeevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.icustays WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.ingredientevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.inputevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.outputevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); +DELETE FROM mimiciv_icu.procedureevents WHERE subject_id NOT IN (SELECT subject_id FROM mimiciv_hosp.patients); + + +-- d_hcpcs +-- d_icd_diagnoses +-- d_icd_procedures +-- d_items +-- d_labitems +-- caregiver +-- provider + +-- to reduce the filesize, the database will need to be exported and reimported +EXPORT DATABASE 'tmp_output'; + +IMPORT DATABASE 'tmp_output'; \ No newline at end of file diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh deleted file mode 100755 index 4ff9fc219..000000000 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2023 MIT Laboratory for Computational Physiology -# Copyright (c) 2021 Thomas Ward -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -yell () { echo "$0: $*" >&2; } -die () { yell "$*"; exit 111; } -try () { "$@" || die "Exiting. Failed to run: \"$*\""; } - -usage () { - die " -USAGE: ./import_duckdb.sh mimic_data_dir [output_db] -WHERE: - mimic_data_dir directory that contains csv.gz or csv files - output_db: optional filename for duckdb file (default: mimic4.db)\ -" -} - -# Print help if requested -echo "$0 $* " | grep -Eq " -h | --help " && usage - -# rename CLI positional args to more friendly variable names -MIMIC_DIR=$1 -# allow optional specification of duckdb name, otherwise default to mimic4.db -OUTFILE=mimic4.db -if [ -n "$2" ]; then - OUTFILE=$2 -fi - - -# basic error checking before running -if [ -z "$MIMIC_DIR" ]; then - yell "Please specify a mimic data directory" - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ ! -d "$MIMIC_DIR" ]; then - yell "Specified directory \"$MIMIC_DIR\" does not exist." - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ -n "$3" ]; then - yell "import_duckdb.sh takes a maximum of two arguments." - die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" -elif [ -s "$OUTFILE" ]; then - yell "File \"$OUTFILE\" already exists." - printf "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " - read -r yn - case $yn in - [Yy]* ) ;; # OK - [Nn]* ) exit;; - [Dd]* ) rm "$OUTFILE";; - * ) die "Unrecognized input.";; - esac -fi - -# we will copy the postgresql create.sql file, and apply regex -# to fix the following issues: -# 1. Remove optional precision value from TIMESTAMP(NN) -> TIMESTAMP -# duckdb does not support this. -export REGEX_TIMESTAMP='s/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' -# 2. Remove NOT NULL constraint from mimiciv_hosp.microbiologyevents.spec_type_desc -# as there is one (!) zero-length string which is treated as a NULL by the import. -export REGEX_SPEC_TYPE='s/spec_type_desc(.+)NOT NULL/spec_type_desc\1/g' -# 3. Remove NOT NULL constraint from mimiciv_hosp.prescriptions.drug -# as there are zero-length strings which are treated as NULLs by the import. -export REGEX_DRUG='s/drug +(VARCHAR.+)NOT NULL/drug \1/g' - -# use sed + above regex to create tables within db -sed -E -e "${REGEX_TIMESTAMP}" ../postgres/create.sql | \ - sed -E -e "${REGEX_SPEC_TYPE}" | \ - sed -E -e "${REGEX_DRUG}" | \ - duckdb "$OUTFILE" - -# goal: get path from find, e.g., ./1.0/icu/d_items -# and return database table name for it, e.g., mimiciv_icu.d_items -make_table_name () { - # strip leading directories (e.g., ./icu/hello.csv.gz -> hello.csv.gz) - BASENAME=${1##*/} - # strip suffix (e.g., hello.csv.gz -> hello; hello.csv -> hello) - TABLE_NAME=${BASENAME%%.*} - # strip basename (e.g., ./icu/hello.csv.gz -> ./icu) - PATHNAME=${1%/*} - # strip leading directories from PATHNAME (e.g. ./icu -> icu) - DIRNAME=${PATHNAME##*/} - TABLE_NAME="mimiciv_$DIRNAME.$TABLE_NAME" -} - - -# load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do - make_table_name "$FILE" - - # skip directories which we do not expect in mimic-iv - # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in - (hosp|icu) ;; # OK - (*) continue; - esac - echo "Loading $FILE .. \c" - OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); -EOSQL - ) - # If the table is missing in the DB, we emit a warning and continue. - # Otherwise, the script repeats the error and exits. - STATUS=$? - if [ $STATUS -ne 0 ]; then - echo "$OUTPUT" | grep -qiE 'table .* does not exist' && { - echo "skipped (table $TABLE_NAME not found)"; - continue; - } - yell "Failed loading $FILE into $TABLE_NAME" - yell "$OUTPUT" - die "Exiting due to load error." - fi - echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/postgres/README.md b/mimic-iv/buildmimic/postgres/README.md index 5ab28c2c4..6b729450f 100644 --- a/mimic-iv/buildmimic/postgres/README.md +++ b/mimic-iv/buildmimic/postgres/README.md @@ -20,6 +20,37 @@ cd mimic-code wget -r -N -c -np --user --ask-password https://physionet.org/files/mimiciv/3.1/ mv physionet.org/files/mimiciv mimiciv && rmdir physionet.org/files && rm physionet.org/robots.txt && rmdir physionet.org createdb mimiciv +PGDATABASE=mimiciv mimic-iv/buildmimic/postgres/build_mimic.sh mimiciv/3.1 +``` + +`build_mimic.sh` creates the schema, loads the data, adds constraints/indexes, +and optionally (default true) derives the [concepts](../../concepts_postgres). + +Compressed (`.csv.gz`) and uncompressed (`.csv`) data are both +detected, and the connection is taken from the standard `PG*` environment +variables. + +It is safe to re-run. Each step is recorded once it completes, and the load +skips tables that already hold rows, so an interrupted build resumes rather than +starting over. This matters most for `chartevents`, which takes by far the +longest to load. Progress is kept in the `mimiciv_build_progress` table. + +Two steps can be turned off: + +```sh +MIMIC_MAKE_CONCEPTS=false MIMIC_VALIDATE=false PGDATABASE=mimiciv \ + mimic-iv/buildmimic/postgres/build_mimic.sh mimiciv/3.1 +``` + +To build in a container instead, see [docker](docker/), which runs this same +script against a containerized PostgreSQL. + +## Running the steps individually + +The build script is a wrapper around the SQL files in this directory, which can +equally be run by hand: + +```sh psql -d mimiciv -f mimic-iv/buildmimic/postgres/create.sql psql -d mimiciv -v ON_ERROR_STOP=1 -v mimic_data_dir=mimiciv/3.1 -f mimic-iv/buildmimic/postgres/load_gz.sql psql -d mimiciv -v ON_ERROR_STOP=1 -v mimic_data_dir=mimiciv/3.1 -f mimic-iv/buildmimic/postgres/constraint.sql diff --git a/mimic-iv/buildmimic/postgres/build_mimic.sh b/mimic-iv/buildmimic/postgres/build_mimic.sh new file mode 100755 index 000000000..fd96987f7 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/build_mimic.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Build MIMIC-IV in PostgreSQL: create the schema, load the data, add the +# constraints and indexes, then derive the concepts. +# +# Usage: +# ./build_mimic.sh +# +# is the directory holding the hosp/ and icu/ subfolders. +# Compressed (.csv.gz) and uncompressed (.csv) data are both supported. +# +# Each step is recorded in a progress table once it completes, so an +# interrupted build will resume where it stopped. +# +# Environment: +# MIMIC_DATA_DIR used when is not given +# MIMIC_MAKE_CONCEPTS derive the concepts into mimiciv_derived (default true) +# MIMIC_VALIDATE check tables against expected row counts (default true) +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +readonly SCRIPT_DIR +readonly CONCEPTS_DIR="${SCRIPT_DIR}/../../concepts_postgres" +readonly PROGRESS_TABLE=public.mimiciv_build_progress + +# load.sql and load_gz.sql set this for their session; the load is run statement +# by statement here, so set it for every connection instead. +export PGCLIENTENCODING=UTF8 + +DATA_DIR="" +LOADER="" + +psql_run() { psql -v ON_ERROR_STOP=1 --quiet "$@"; } +psql_val() { psql -v ON_ERROR_STOP=1 -Atq -c "$1"; } + +# Lowercase string before checking value +is_true() { [ "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" = "true" ]; } + +usage() { + echo "Usage: $(basename "$0") " >&2 + echo " the directory containing the hosp/ and icu/ subfolders" >&2 +} + +# Completed steps are recorded in a table located on the public schema. +step_done() { + [ "$(psql_val "SELECT EXISTS (SELECT 1 FROM ${PROGRESS_TABLE} WHERE step = '$1')")" = "t" ] +} + +run_step() { + local step=$1 + shift + if step_done "${step}"; then + echo "== ${step}: already done, skipping" + return + fi + echo "== ${step}: starting" + "$@" + psql_run -c "INSERT INTO ${PROGRESS_TABLE} (step) VALUES ('${step}') ON CONFLICT DO NOTHING" + echo "== ${step}: done" +} + +create_schema() { + psql_run -f "${SCRIPT_DIR}/create.sql" +} + +# To reduce redundancy, this script reads the \COPY statements to determine +# the tables to build, and builds them in order. +parse_copy_statements() { + awk -F' ' ' + /^\\cd / { + dir = $2 + if (dir ~ /^:/) { cur = "" } # \cd :mimic_data_dir + else { sub(/^\.\.\//, "", dir); cur = dir } # \cd hosp, \cd ../icu + next + } + /^\\COPY / { print cur "\t" $2 "\t" $0 } + ' "${SCRIPT_DIR}/${LOADER}" +} + +# A \COPY is a single statement, so a table is either fully loaded or empty. +# Even a single row indicates completion of the data load. +table_has_rows() { + [ "$(psql_val "SELECT EXISTS (SELECT 1 FROM $1 LIMIT 1)")" = "t" ] +} + +load_data() { + local subdir table statement + while IFS=$'\t' read -r subdir table statement; do + if table_has_rows "${table}"; then + echo " ${table}: already loaded, skipping" + continue + fi + echo " ${table}: loading" + (cd "${DATA_DIR}/${subdir}" && printf '%s\n' "${statement}" | psql_run) + done < <(parse_copy_statements) +} + +add_constraints() { + psql_run -f "${SCRIPT_DIR}/constraint.sql" +} + +build_indexes() { + psql_run -f "${SCRIPT_DIR}/index.sql" +} + +make_concepts() { + # postgres-make-concepts.sql pulls in the individual concepts with \i, which + # resolves relative to the working directory rather than to the script. + cd "${CONCEPTS_DIR}" + psql_run -f postgres-make-concepts.sql +} + +validate() { + # default validate the full DB + local script=validate.sql + + # The demo is a 100 patient subset of MIMIC-IV, so it has its own set of + # expected row counts. + if [ "$(psql_val 'SELECT count(*) FROM mimiciv_hosp.patients')" -eq 100 ]; then + echo " 100 patients found, validating against the MIMIC-IV demo counts" + script=validate_demo.sql + fi + + local output + output=$(psql -v ON_ERROR_STOP=1 -f "${SCRIPT_DIR}/${script}") + echo "${output}" + + # Mismatches indicate (1) failure to load (zero rows) or (2) mismatch mimic version + if echo "${output}" | grep -q FAILED; then + echo "WARNING: some tables do not have the expected number of rows." >&2 + fi +} + +resolve_data_dir() { + local dir=${1:-${MIMIC_DATA_DIR:-}} + + if [ -z "${dir}" ]; then + usage + exit 1 + fi + if [ ! -d "${dir}/hosp" ] || [ ! -d "${dir}/icu" ]; then + echo "ERROR: ${dir} must contain the hosp/ and icu/ subfolders of MIMIC-IV." >&2 + exit 1 + fi + + DATA_DIR=$(cd -- "${dir}" && pwd) +} + +# Prefer the compressed data (the default distribution format). +select_loader() { + if [ -n "$(find "${DATA_DIR}/hosp" -maxdepth 1 -name '*.csv.gz' -print -quit)" ]; then + LOADER=load_gz.sql + elif [ -n "$(find "${DATA_DIR}/hosp" -maxdepth 1 -name '*.csv' -print -quit)" ]; then + LOADER=load.sql + else + echo "ERROR: no .csv or .csv.gz files found in ${DATA_DIR}/hosp." >&2 + exit 1 + fi + echo "Loading from ${DATA_DIR} using ${LOADER}" +} + +main() { + case "${1:-}" in + -h|--help) usage; exit 0 ;; + esac + + resolve_data_dir "${1:-}" + select_loader + + psql_run -c "CREATE TABLE IF NOT EXISTS ${PROGRESS_TABLE} ( + step text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() + )" + + run_step create create_schema + run_step load load_data + run_step constraint add_constraints + run_step index build_indexes + + if is_true "${MIMIC_MAKE_CONCEPTS:-true}"; then + run_step concepts make_concepts + else + echo "== concepts: MIMIC_MAKE_CONCEPTS is not 'true', skipping" + fi + + if is_true "${MIMIC_VALIDATE:-true}"; then + run_step validate validate + else + echo "== validate: MIMIC_VALIDATE is not 'true', skipping" + fi + + echo "MIMIC-IV build complete." +} + +main "$@" diff --git a/mimic-iv/buildmimic/postgres/docker/.env.example b/mimic-iv/buildmimic/postgres/docker/.env.example new file mode 100644 index 000000000..7750b3f91 --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/.env.example @@ -0,0 +1,24 @@ +# Copy to .env and edit as needed: cp .env.example .env + +# PostgreSQL version to build against. 16, 17 and 18 are supported. +PG_VERSION=16 + +# Credentials +POSTGRES_DB=mimiciv +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres + +# Host port to publish. Change it if you already run postgres on 5432. +POSTGRES_PORT=5432 + +# Directory holding the MIMIC-IV .csv.gz files, containing hosp/ and icu/ +# subfolders. Populate it by running ./download_data.sh +MIMIC_DATA_DIR=./mimic-data + +# Derive the concepts from mimic-iv/concepts_postgres once the data is loaded. +# Set to false to load the raw tables only. +MIMIC_MAKE_CONCEPTS=true + +# Check the loaded tables against the expected row counts. The demo and the full +# dataset are detected automatically. +MIMIC_VALIDATE=true diff --git a/mimic-iv/buildmimic/postgres/docker/Dockerfile b/mimic-iv/buildmimic/postgres/docker/Dockerfile index 2dd098fb3..d7368334f 100644 --- a/mimic-iv/buildmimic/postgres/docker/Dockerfile +++ b/mimic-iv/buildmimic/postgres/docker/Dockerfile @@ -1,4 +1,18 @@ -FROM postgres -COPY ./src/ /docker-entrypoint-initdb.d/ -WORKDIR /app -EXPOSE 5432 +# Image for the one-shot job that builds MIMIC-IV into the database. The +# database itself runs the stock postgres image; see docker-compose.yml. +# +# The build context is the mimic-iv/ directory, and the repository layout is +# kept intact inside the image so that build_mimic.sh finds the SQL beside it +# and the concepts two directories up. Nothing is duplicated. +# +# PostgreSQL 16, 17 and 18 are all supported. +ARG PG_VERSION=16 +FROM postgres:${PG_VERSION} + +COPY buildmimic/postgres/*.sql buildmimic/postgres/build_mimic.sh /mimic/buildmimic/postgres/ +COPY concepts_postgres/ /mimic/concepts_postgres/ + +# Replaces the postgres image's own entrypoint: this container is a psql client, +# not a server. The data directory is passed as the command. +ENTRYPOINT ["/mimic/buildmimic/postgres/build_mimic.sh"] +CMD ["/data"] diff --git a/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore b/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..e454902cd --- /dev/null +++ b/mimic-iv/buildmimic/postgres/docker/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +# The build context is mimic-iv/, which holds large demo databases and the +# notebooks. Exclude everything, then add back only what the image needs. +* +!buildmimic/postgres/*.sql +!buildmimic/postgres/build_mimic.sh +!concepts_postgres/ diff --git a/mimic-iv/buildmimic/postgres/docker/README.md b/mimic-iv/buildmimic/postgres/docker/README.md index 242cdeb7a..2dd6b8f08 100644 --- a/mimic-iv/buildmimic/postgres/docker/README.md +++ b/mimic-iv/buildmimic/postgres/docker/README.md @@ -1,37 +1,101 @@ -# Getting MIMIC-IV up and running with Postgres and Docker -Use these scripts to quickly get setup with a containerized version of MIMIC-IV +# MIMIC-IV with PostgreSQL and Docker -## Steps -1. Install Docker -2. Download mimic data from PhysioNet by running the `download_data.sh` script. this will store the files in ./mimic-data/ -3. Edit `POSTGRES_USER` and `POSTGRES_PASSWORD` as needed. Optionally, you can - change the host paths for the volume mounts if you would like to change where - the data are stored on the host machine. -4. Build and run the container by running `docker compose up`. This may take awhile. +Build a containerized PostgreSQL database containing MIMIC-IV. +The docker container uses build scripts in the parent folder. +It creates the schema, loads the data, adds the constraints and indexes, +(optionally, default true) derives the concepts from +[concepts_postgres](../../../concepts_postgres), and validates the result. -Once complete you should have a containerized postgres instance containing MIMIC-IV data in the `mimiciv` database. +There are two services. `mimic-db` is a stock postgres image holding the data. +`mimic-build` is a single job that populates it and then exits. They share the +PostgreSQL unix socket, so the bulk `COPY` does not cross the container network. -## Using the database +## Requirements + +* Docker, with Compose v2. +* The dataset (`../../download_data.sh` is a convenience for downloading data from PhysioNet) +* Disk space. The compressed download is about 10 GB, but the loaded database is + considerably larger: roughly 140 GB for `mimiciv_hosp` and `mimiciv_icu` with + their indexes, plus about 10 GB if you build the concepts. On macOS and + Windows this space is taken from the Docker Desktop virtual disk, so raise + that limit in Settings first. + +## Quickstart + +```bash +# 1. Configure. The defaults should work as-is. +# Make sure MIMIC_DATA_DIR has the hosp/ and icu/ subfolders. +cp .env.example .env + +# 2. Download the data, if you do not already have it (about 10 GB). +../../download_data.sh + +# 3. Build. Runs in the foreground so you can watch it; add -d to detach. +docker compose up +``` + +Loading takes several hours for the full dataset, plus about an hour for the +concepts. The database accepts connections throughout, so an empty or partial +result simply means the build is still running. Watch it with: ```bash -# spin up a container and detach -docker compose up -d +docker compose logs -f mimic-build +``` + +## Notes -# you should now see a running container...e.g docker-mimic-db-1 -docker ps +### `download_data.sh` -# if you have psql installed on the host machine you can do...password required -psql -U -d mimiciv -h localhost +`download_data.sh` is a convenience wrapper around `wget`. If you already have +MIMIC-IV, skip it and point `MIMIC_DATA_DIR` in `.env` at your copy. Any +directory containing the `hosp/` and `icu/` subfolders will work, including the +[demo dataset](https://physionet.org/content/mimic-iv-demo/), which is a useful +way to try this out without downloading the full 10 GB. -# if you do not have psql installed on host machine you can do...password not -# required -docker exec -it docker-mimic-db-1 psql -U -d mimiciv +### If the build is interrupted -# psql on host machine...run profile script -psql -U -d mimiciv -h localhost -f ../validate.sql +Run `docker compose up` again. The build records each step as it completes and +skips tables that are already populated, so it resumes rather than starting +over. This matters mostly for `chartevents`, the largest table. + +Resuming is safe because a `\COPY` is a single statement: a table is either fully loaded or empty. Progress is tracked in the +`mimiciv_build_progress` table, which you can inspect: + +```bash +docker compose exec mimic-db psql -U postgres -d mimiciv -c 'TABLE mimiciv_build_progress' +``` + +To discard everything and start clean: + +```bash +docker compose down -v +docker compose up --build +``` + +Use `--build` whenever you change the SQL or the postgres version, otherwise +Compose reuses the existing image. + +### Settings + +All are set in `.env`; see `.env.example` for the defaults. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `PG_VERSION` | `16` | PostgreSQL major version. 16, 17 and 18 are supported. | +| `POSTGRES_DB` | `mimiciv` | Database name. | +| `POSTGRES_USER` | `postgres` | Database user. | +| `POSTGRES_PASSWORD` | `postgres` | Database password. | +| `POSTGRES_PORT` | `5432` | Host port to publish. | +| `MIMIC_DATA_DIR` | `./mimic-data` | Directory holding `hosp/` and `icu/`. | +| `MIMIC_MAKE_CONCEPTS` | `true` | Derive the concepts into `mimiciv_derived`. | +| `MIMIC_VALIDATE` | `true` | Check the loaded tables against expected row counts. | + +## Using the database + +```bash +# from the host, if you have psql installed +psql -h localhost -U postgres -d mimiciv -# psql not on host machine...need to mv validate into mount director so it can -# be accessed from container -cp ../validate.sql ./mimic-data/ -docker exec -it docker-mimic-db-1 psql -U -d mimiciv -f /data/validate.sql +# otherwise, from inside the container +docker compose exec mimic-db psql -U postgres -d mimiciv ``` diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml index 418689ce6..8da8bbfcf 100644 --- a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -1,30 +1,51 @@ -# Use postgres/example user/password credentials -version: '3.9' services: mimic-db: - image: mimiciv - build: . - restart: always + container_name: mimic-db + image: postgres:${PG_VERSION:-16} ports: - # you may need to change the host port if you have another postgres - # instance running on the host machine. - - 5432:5432 + - "${POSTGRES_PORT:-5432}:5432" + environment: + POSTGRES_DB: ${POSTGRES_DB:-mimiciv} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} volumes: - # this mounts the mimic csv files to the /data/ directory within the - # container. - - ./mimic-data/:/data/ - # this maps a directory on the host machine (i.e. ./mimic-db/) to the - # container so that the data in the database can persist after container - # has been shut down. - - /data/mimic-iv-db/:/var/lib/postgresql/data/ + # Mounted at /var/lib/postgresql rather than the data directory itself. + # PostgreSQL 18 moved the data directory into a per-version subfolder. + - mimic-pgdata:/var/lib/postgresql + # Shares the unix socket with mimic-build, so the bulk COPY does not have + # to cross the container network. + - mimic-socket:/var/run/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 + + # Loads the data and derives the concepts, then exits. + # Re-running it resumes an interrupted build. + mimic-build: + build: + # mimic-iv/, so the build can reach buildmimic/postgres and concepts_postgres + context: ../../.. + dockerfile: buildmimic/postgres/docker/Dockerfile + args: + PG_VERSION: ${PG_VERSION:-16} + depends_on: + mimic-db: + condition: service_healthy environment: - # this will be the name of the database within Postgres. - POSTGRES_DB: physionet-data - # set this to the same user that owns the directory storing the .csv data - # (hint: check ls -l ./mimic-data/) this should be the user of host - # machine. this will also be the user you will use to authenticate with - # postgres - POSTGRES_USER: mlbernauer - # this will be the password for authenticating with postgres...set to - # whatever you want - POSTGRES_PASSWORD: pass123 + # Connects over the shared unix socket rather than TCP. + PGHOST: /var/run/postgresql + PGDATABASE: ${POSTGRES_DB:-mimiciv} + PGUSER: ${POSTGRES_USER:-postgres} + PGPASSWORD: ${POSTGRES_PASSWORD:-postgres} + MIMIC_MAKE_CONCEPTS: ${MIMIC_MAKE_CONCEPTS:-true} + MIMIC_VALIDATE: ${MIMIC_VALIDATE:-true} + volumes: + # The .csv.gz files, read-only. Must contain hosp/ and icu/ subfolders. + - ${MIMIC_DATA_DIR:-./mimic-data}:/data:ro + - mimic-socket:/var/run/postgresql + +volumes: + mimic-pgdata: + mimic-socket: diff --git a/mimic-iv/buildmimic/postgres/docker/download_data.sh b/mimic-iv/buildmimic/postgres/docker/download_data.sh deleted file mode 100755 index 0819330bb..000000000 --- a/mimic-iv/buildmimic/postgres/docker/download_data.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -read -p "Enter PhysioNet username: " USERNAME -echo "Downloading MIMIV-IV data to ./mimic-data/" -wget -P -A gz,txt ./mimic-data/ -r -N -c -nd -np --user ${USERNAME} --ask-password https://physionet.org/files/mimiciv/2.2/ diff --git a/mimic-iv/buildmimic/postgres/docker/src/01_create.sql b/mimic-iv/buildmimic/postgres/docker/src/01_create.sql deleted file mode 100644 index 14080d22a..000000000 --- a/mimic-iv/buildmimic/postgres/docker/src/01_create.sql +++ /dev/null @@ -1,519 +0,0 @@ -------------------------------------------- --- Create the tables and MIMIC-IV schema -- -------------------------------------------- - ----------------------- --- Creating schemas -- ----------------------- - -DROP SCHEMA IF EXISTS mimiciv_hosp CASCADE; -CREATE SCHEMA mimiciv_hosp; -DROP SCHEMA IF EXISTS mimiciv_icu CASCADE; -CREATE SCHEMA mimiciv_icu; -DROP SCHEMA IF EXISTS mimiciv_derived CASCADE; -CREATE SCHEMA mimiciv_derived; - ---------------------- --- Creating tables -- ---------------------- - --- hosp schema - -DROP TABLE IF EXISTS mimiciv_hosp.admissions; -CREATE TABLE mimiciv_hosp.admissions -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - admittime TIMESTAMP NOT NULL, - dischtime TIMESTAMP, - deathtime TIMESTAMP, - admission_type VARCHAR(40) NOT NULL, - admit_provider_id VARCHAR(10), - admission_location VARCHAR(60), - discharge_location VARCHAR(60), - insurance VARCHAR(255), - language VARCHAR(10), - marital_status VARCHAR(30), - race VARCHAR(80), - edregtime TIMESTAMP, - edouttime TIMESTAMP, - hospital_expire_flag SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_hosp.d_hcpcs; -CREATE TABLE mimiciv_hosp.d_hcpcs -( - code CHAR(5) NOT NULL, - category SMALLINT, - long_description TEXT, - short_description VARCHAR(180) -); - -DROP TABLE IF EXISTS mimiciv_hosp.diagnoses_icd; -CREATE TABLE mimiciv_hosp.diagnoses_icd -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - seq_num INTEGER NOT NULL, - icd_code CHAR(7), - icd_version SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_hosp.d_icd_diagnoses; -CREATE TABLE mimiciv_hosp.d_icd_diagnoses -( - icd_code CHAR(7) NOT NULL, - icd_version SMALLINT NOT NULL, - long_title VARCHAR(255) -); - -DROP TABLE IF EXISTS mimiciv_hosp.d_icd_procedures; -CREATE TABLE mimiciv_hosp.d_icd_procedures -( - icd_code CHAR(7) NOT NULL, - icd_version SMALLINT NOT NULL, - long_title VARCHAR(222) -); - -DROP TABLE IF EXISTS mimiciv_hosp.d_labitems; -CREATE TABLE mimiciv_hosp.d_labitems -( - itemid INTEGER NOT NULL, - label VARCHAR(50), - fluid VARCHAR(50), - category VARCHAR(50) -); - -DROP TABLE IF EXISTS mimiciv_hosp.drgcodes; -CREATE TABLE mimiciv_hosp.drgcodes -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - drg_type VARCHAR(4), - drg_code VARCHAR(10) NOT NULL, - description VARCHAR(195), - drg_severity SMALLINT, - drg_mortality SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_hosp.emar_detail; -CREATE TABLE mimiciv_hosp.emar_detail -( - subject_id INTEGER NOT NULL, - emar_id VARCHAR(25) NOT NULL, - emar_seq INTEGER NOT NULL, - parent_field_ordinal VARCHAR(10), - administration_type VARCHAR(50), - pharmacy_id INTEGER, - barcode_type VARCHAR(4), - reason_for_no_barcode TEXT, - complete_dose_not_given VARCHAR(5), - dose_due VARCHAR(100), - dose_due_unit VARCHAR(50), - dose_given VARCHAR(255), - dose_given_unit VARCHAR(50), - will_remainder_of_dose_be_given VARCHAR(5), - product_amount_given VARCHAR(30), - product_unit VARCHAR(30), - product_code VARCHAR(30), - product_description VARCHAR(255), - product_description_other VARCHAR(255), - prior_infusion_rate VARCHAR(40), - infusion_rate VARCHAR(40), - infusion_rate_adjustment VARCHAR(50), - infusion_rate_adjustment_amount VARCHAR(30), - infusion_rate_unit VARCHAR(30), - route VARCHAR(10), - infusion_complete VARCHAR(1), - completion_interval VARCHAR(50), - new_iv_bag_hung VARCHAR(1), - continued_infusion_in_other_location VARCHAR(1), - restart_interval VARCHAR(2305), - side VARCHAR(10), - site VARCHAR(255), - non_formulary_visual_verification VARCHAR(1) -); - -DROP TABLE IF EXISTS mimiciv_hosp.emar; -CREATE TABLE mimiciv_hosp.emar -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER, - emar_id VARCHAR(25) NOT NULL, - emar_seq INTEGER NOT NULL, - poe_id VARCHAR(25) NOT NULL, - pharmacy_id INTEGER, - enter_provider_id VARCHAR(10), - charttime TIMESTAMP NOT NULL, - medication TEXT, - event_txt VARCHAR(100), - scheduletime TIMESTAMP, - storetime TIMESTAMP NOT NULL -); - -DROP TABLE IF EXISTS mimiciv_hosp.hcpcsevents; -CREATE TABLE mimiciv_hosp.hcpcsevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - chartdate DATE, - hcpcs_cd CHAR(5) NOT NULL, - seq_num INTEGER NOT NULL, - short_description VARCHAR(180) -); - -DROP TABLE IF EXISTS mimiciv_hosp.labevents; -CREATE TABLE mimiciv_hosp.labevents -( - labevent_id INTEGER NOT NULL, - subject_id INTEGER NOT NULL, - hadm_id INTEGER, - specimen_id INTEGER NOT NULL, - itemid INTEGER NOT NULL, - order_provider_id VARCHAR(10), - charttime TIMESTAMP(0), - storetime TIMESTAMP(0), - value VARCHAR(200), - valuenum DOUBLE PRECISION, - valueuom VARCHAR(20), - ref_range_lower DOUBLE PRECISION, - ref_range_upper DOUBLE PRECISION, - flag VARCHAR(10), - priority VARCHAR(7), - comments TEXT -); - -DROP TABLE IF EXISTS mimiciv_hosp.microbiologyevents; -CREATE TABLE mimiciv_hosp.microbiologyevents -( - microevent_id INTEGER NOT NULL, - subject_id INTEGER NOT NULL, - hadm_id INTEGER, - micro_specimen_id INTEGER NOT NULL, - order_provider_id VARCHAR(10), - chartdate TIMESTAMP(0) NOT NULL, - charttime TIMESTAMP(0), - spec_itemid INTEGER NOT NULL, - spec_type_desc VARCHAR(100) NOT NULL, - test_seq INTEGER NOT NULL, - storedate TIMESTAMP(0), - storetime TIMESTAMP(0), - test_itemid INTEGER, - test_name VARCHAR(100), - org_itemid INTEGER, - org_name VARCHAR(100), - isolate_num SMALLINT, - quantity VARCHAR(50), - ab_itemid INTEGER, - ab_name VARCHAR(30), - dilution_text VARCHAR(10), - dilution_comparison VARCHAR(20), - dilution_value DOUBLE PRECISION, - interpretation VARCHAR(5), - comments TEXT -); - -DROP TABLE IF EXISTS mimiciv_hosp.omr; -CREATE TABLE mimiciv_hosp.omr -( - subject_id INTEGER NOT NULL, - chartdate DATE NOT NULL, - seq_num INTEGER NOT NULL, - result_name VARCHAR(100) NOT NULL, - result_value TEXT NOT NULL -); - -DROP TABLE IF EXISTS mimiciv_hosp.patients; -CREATE TABLE mimiciv_hosp.patients -( - subject_id INTEGER NOT NULL, - gender CHAR(1) NOT NULL, - anchor_age SMALLINT, - anchor_year SMALLINT NOT NULL, - anchor_year_group VARCHAR(20) NOT NULL, - dod DATE -); - -DROP TABLE IF EXISTS mimiciv_hosp.pharmacy; -CREATE TABLE mimiciv_hosp.pharmacy -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - pharmacy_id INTEGER NOT NULL, - poe_id VARCHAR(25), - starttime TIMESTAMP(3), - stoptime TIMESTAMP(3), - medication TEXT, - proc_type VARCHAR(50) NOT NULL, - status VARCHAR(50), - entertime TIMESTAMP(3) NOT NULL, - verifiedtime TIMESTAMP(3), - route VARCHAR(50), - frequency VARCHAR(50), - disp_sched VARCHAR(255), - infusion_type VARCHAR(15), - sliding_scale VARCHAR(1), - lockout_interval VARCHAR(50), - basal_rate REAL, - one_hr_max VARCHAR(10), - doses_per_24_hrs REAL, - duration REAL, - duration_interval VARCHAR(50), - expiration_value INTEGER, - expiration_unit VARCHAR(50), - expirationdate TIMESTAMP(3), - dispensation VARCHAR(50), - fill_quantity VARCHAR(50) -); - -DROP TABLE IF EXISTS mimiciv_hosp.poe_detail; -CREATE TABLE mimiciv_hosp.poe_detail -( - poe_id VARCHAR(25) NOT NULL, - poe_seq INTEGER NOT NULL, - subject_id INTEGER NOT NULL, - field_name VARCHAR(255) NOT NULL, - field_value TEXT -); - -DROP TABLE IF EXISTS mimiciv_hosp.poe; -CREATE TABLE mimiciv_hosp.poe -( - poe_id VARCHAR(25) NOT NULL, - poe_seq INTEGER NOT NULL, - subject_id INTEGER NOT NULL, - hadm_id INTEGER, - ordertime TIMESTAMP(0) NOT NULL, - order_type VARCHAR(25) NOT NULL, - order_subtype VARCHAR(50), - transaction_type VARCHAR(15), - discontinue_of_poe_id VARCHAR(25), - discontinued_by_poe_id VARCHAR(25), - order_provider_id VARCHAR(10), - order_status VARCHAR(15) -); - -DROP TABLE IF EXISTS mimiciv_hosp.prescriptions; -CREATE TABLE mimiciv_hosp.prescriptions -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - pharmacy_id INTEGER NOT NULL, - poe_id VARCHAR(25), - poe_seq INTEGER, - order_provider_id VARCHAR(10), - starttime TIMESTAMP(3), - stoptime TIMESTAMP(3), - drug_type VARCHAR(20) NOT NULL, - drug VARCHAR(255) NOT NULL, - formulary_drug_cd VARCHAR(50), - gsn VARCHAR(255), - ndc VARCHAR(25), - prod_strength VARCHAR(255), - form_rx VARCHAR(25), - dose_val_rx VARCHAR(100), - dose_unit_rx VARCHAR(50), - form_val_disp VARCHAR(50), - form_unit_disp VARCHAR(50), - doses_per_24_hrs REAL, - route VARCHAR(50) -); - -DROP TABLE IF EXISTS mimiciv_hosp.procedures_icd; -CREATE TABLE mimiciv_hosp.procedures_icd -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - seq_num INTEGER NOT NULL, - chartdate DATE NOT NULL, - icd_code VARCHAR(7), - icd_version SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_hosp.provider; -CREATE TABLE mimiciv_hosp.provider -( - provider_id VARCHAR(10) NOT NULL -); - -DROP TABLE IF EXISTS mimiciv_hosp.services; -CREATE TABLE mimiciv_hosp.services -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - transfertime TIMESTAMP NOT NULL, - prev_service VARCHAR(10), - curr_service VARCHAR(10) -); - -DROP TABLE IF EXISTS mimiciv_hosp.transfers; -CREATE TABLE mimiciv_hosp.transfers -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER, - transfer_id INTEGER NOT NULL, - eventtype VARCHAR(10), - careunit VARCHAR(255), - intime TIMESTAMP, - outtime TIMESTAMP -); - --- icu schema - -DROP TABLE IF EXISTS mimiciv_icu.caregiver; -CREATE TABLE mimiciv_icu.caregiver -( - caregiver_id INTEGER NOT NULL -); - -DROP TABLE IF EXISTS mimiciv_icu.chartevents; -CREATE TABLE mimiciv_icu.chartevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER NOT NULL, - caregiver_id INTEGER, - charttime TIMESTAMP NOT NULL, - storetime TIMESTAMP, - itemid INTEGER NOT NULL, - value VARCHAR(200), - valuenum FLOAT, - valueuom VARCHAR(20), - warning SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_icu.datetimeevents; -CREATE TABLE mimiciv_icu.datetimeevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER NOT NULL, - caregiver_id INTEGER, - charttime TIMESTAMP NOT NULL, - storetime TIMESTAMP, - itemid INTEGER NOT NULL, - value TIMESTAMP NOT NULL, - valueuom VARCHAR(20), - warning SMALLINT -); - -DROP TABLE IF EXISTS mimiciv_icu.d_items; -CREATE TABLE mimiciv_icu.d_items -( - itemid INTEGER NOT NULL, - label VARCHAR(100) NOT NULL, - abbreviation VARCHAR(50) NOT NULL, - linksto VARCHAR(30) NOT NULL, - category VARCHAR(50) NOT NULL, - unitname VARCHAR(50), - param_type VARCHAR(20) NOT NULL, - lownormalvalue FLOAT, - highnormalvalue FLOAT -); - -DROP TABLE IF EXISTS mimiciv_icu.icustays; -CREATE TABLE mimiciv_icu.icustays -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER NOT NULL, - first_careunit VARCHAR(255), - last_careunit VARCHAR(255), - intime TIMESTAMP, - outtime TIMESTAMP, - los FLOAT -); - -DROP TABLE IF EXISTS mimiciv_icu.ingredientevents; -CREATE TABLE mimiciv_icu.ingredientevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER, - caregiver_id INTEGER, - starttime TIMESTAMP NOT NULL, - endtime TIMESTAMP NOT NULL, - storetime TIMESTAMP, - itemid INTEGER NOT NULL, - amount FLOAT, - amountuom VARCHAR(20), - rate FLOAT, - rateuom VARCHAR(20), - orderid INTEGER NOT NULL, - linkorderid INTEGER, - statusdescription VARCHAR(20), - originalamount FLOAT, - originalrate FLOAT -); - -DROP TABLE IF EXISTS mimiciv_icu.inputevents; -CREATE TABLE mimiciv_icu.inputevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER, - caregiver_id INTEGER, - starttime TIMESTAMP NOT NULL, - endtime TIMESTAMP NOT NULL, - storetime TIMESTAMP, - itemid INTEGER NOT NULL, - amount FLOAT, - amountuom VARCHAR(20), - rate FLOAT, - rateuom VARCHAR(20), - orderid INTEGER NOT NULL, - linkorderid INTEGER, - ordercategoryname VARCHAR(50), - secondaryordercategoryname VARCHAR(50), - ordercomponenttypedescription VARCHAR(100), - ordercategorydescription VARCHAR(30), - patientweight FLOAT, - totalamount FLOAT, - totalamountuom VARCHAR(50), - isopenbag SMALLINT, - continueinnextdept SMALLINT, - statusdescription VARCHAR(20), - originalamount FLOAT, - originalrate FLOAT -); - -DROP TABLE IF EXISTS mimiciv_icu.outputevents; -CREATE TABLE mimiciv_icu.outputevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER NOT NULL, - caregiver_id INTEGER, - charttime TIMESTAMP(3) NOT NULL, - storetime TIMESTAMP(3) NOT NULL, - itemid INTEGER NOT NULL, - value FLOAT NOT NULL, - valueuom VARCHAR(20) -); - -DROP TABLE IF EXISTS mimiciv_icu.procedureevents; -CREATE TABLE mimiciv_icu.procedureevents -( - subject_id INTEGER NOT NULL, - hadm_id INTEGER NOT NULL, - stay_id INTEGER NOT NULL, - caregiver_id INTEGER, - starttime TIMESTAMP NOT NULL, - endtime TIMESTAMP NOT NULL, - storetime TIMESTAMP NOT NULL, - itemid INTEGER NOT NULL, - value FLOAT, - valueuom VARCHAR(20), - location VARCHAR(100), - locationcategory VARCHAR(50), - orderid INTEGER, - linkorderid INTEGER, - ordercategoryname VARCHAR(50), - ordercategorydescription VARCHAR(30), - patientweight FLOAT, - isopenbag SMALLINT, - continueinnextdept SMALLINT, - statusdescription VARCHAR(20), - originalamount FLOAT, - originalrate FLOAT -); diff --git a/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql b/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql deleted file mode 100644 index 01bff843e..000000000 --- a/mimic-iv/buildmimic/postgres/docker/src/02_load_gz.sql +++ /dev/null @@ -1,37 +0,0 @@ ------------------------------------------ --- Load data into the MIMIC-IV schemas -- ------------------------------------------ - -SET CLIENT_ENCODING TO 'utf8'; -\cd /data -\COPY mimiciv_hosp.admissions FROM PROGRAM 'gzip -dc admissions.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.d_hcpcs FROM PROGRAM 'gzip -dc d_hcpcs.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.diagnoses_icd FROM PROGRAM 'gzip -dc diagnoses_icd.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.d_icd_diagnoses FROM PROGRAM 'gzip -dc d_icd_diagnoses.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.d_icd_procedures FROM PROGRAM 'gzip -dc d_icd_procedures.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.d_labitems FROM PROGRAM 'gzip -dc d_labitems.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.drgcodes FROM PROGRAM 'gzip -dc drgcodes.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.emar_detail FROM PROGRAM 'gzip -dc emar_detail.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.emar FROM PROGRAM 'gzip -dc emar.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.hcpcsevents FROM PROGRAM 'gzip -dc hcpcsevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.labevents FROM PROGRAM 'gzip -dc labevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.microbiologyevents FROM PROGRAM 'gzip -dc microbiologyevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.omr FROM PROGRAM 'gzip -dc omr.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.patients FROM PROGRAM 'gzip -dc patients.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.pharmacy FROM PROGRAM 'gzip -dc pharmacy.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.poe_detail FROM PROGRAM 'gzip -dc poe_detail.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.poe FROM PROGRAM 'gzip -dc poe.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.prescriptions FROM PROGRAM 'gzip -dc prescriptions.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.procedures_icd FROM PROGRAM 'gzip -dc procedures_icd.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.provider FROM PROGRAM 'gzip -dc provider.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.services FROM PROGRAM 'gzip -dc services.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_hosp.transfers FROM PROGRAM 'gzip -dc transfers.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.caregiver FROM PROGRAM 'gzip -dc caregiver.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.chartevents FROM PROGRAM 'gzip -dc chartevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.datetimeevents FROM PROGRAM 'gzip -dc datetimeevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.d_items FROM PROGRAM 'gzip -dc d_items.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.icustays FROM PROGRAM 'gzip -dc icustays.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.ingredientevents FROM PROGRAM 'gzip -dc ingredientevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.inputevents FROM PROGRAM 'gzip -dc inputevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.outputevents FROM PROGRAM 'gzip -dc outputevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; -\COPY mimiciv_icu.procedureevents FROM PROGRAM 'gzip -dc procedureevents.csv.gz' DELIMITER ',' CSV HEADER NULL ''; diff --git a/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql b/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql deleted file mode 100644 index 6c63c2efd..000000000 --- a/mimic-iv/buildmimic/postgres/docker/src/03_constrant.sql +++ /dev/null @@ -1,551 +0,0 @@ ---------------------------- ---------------------------- --- Creating Primary Keys -- ---------------------------- ---------------------------- - ----------- --- hosp -- ----------- - --- admissions - -ALTER TABLE mimiciv_hosp.admissions DROP CONSTRAINT IF EXISTS admissions_pk CASCADE; -ALTER TABLE mimiciv_hosp.admissions -ADD CONSTRAINT admissions_pk - PRIMARY KEY (hadm_id); - --- d_hcpcs - -ALTER TABLE mimiciv_hosp.d_hcpcs DROP CONSTRAINT IF EXISTS d_hcpcs_pk CASCADE; -ALTER TABLE mimiciv_hosp.d_hcpcs -ADD CONSTRAINT d_hcpcs_pk - PRIMARY KEY (code); - --- diagnoses_icd - -ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_pk CASCADE; -ALTER TABLE mimiciv_hosp.diagnoses_icd -ADD CONSTRAINT diagnoses_icd_pk - PRIMARY KEY (hadm_id, seq_num, icd_code, icd_version); - --- d_icd_diagnoses - -ALTER TABLE mimiciv_hosp.d_icd_diagnoses DROP CONSTRAINT IF EXISTS d_icd_diagnoses_pk CASCADE; -ALTER TABLE mimiciv_hosp.d_icd_diagnoses -ADD CONSTRAINT d_icd_diagnoses_pk - PRIMARY KEY (icd_code, icd_version); - --- d_icd_procedures - -ALTER TABLE mimiciv_hosp.d_icd_procedures DROP CONSTRAINT IF EXISTS d_icd_procedures_pk CASCADE; -ALTER TABLE mimiciv_hosp.d_icd_procedures -ADD CONSTRAINT d_icd_procedures_pk - PRIMARY KEY (icd_code, icd_version); - --- d_labitems - -ALTER TABLE mimiciv_hosp.d_labitems DROP CONSTRAINT IF EXISTS d_labitems_pk CASCADE; -ALTER TABLE mimiciv_hosp.d_labitems -ADD CONSTRAINT d_labitems_pk - PRIMARY KEY (itemid); - --- emar_detail - --- ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_pk; --- ALTER TABLE mimiciv_hosp.emar_detail --- ADD CONSTRAINT emar_detail_pk --- PRIMARY KEY (emar_id, parent_field_ordinal); - --- emar - -ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_pk CASCADE; -ALTER TABLE mimiciv_hosp.emar -ADD CONSTRAINT emar_pk - PRIMARY KEY (emar_id); - --- hcpcsevents - -ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_pk CASCADE; -ALTER TABLE mimiciv_hosp.hcpcsevents -ADD CONSTRAINT hcpcsevents_pk - PRIMARY KEY (hadm_id, hcpcs_cd, seq_num); - --- labevents - -ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_pk CASCADE; -ALTER TABLE mimiciv_hosp.labevents -ADD CONSTRAINT labevents_pk - PRIMARY KEY (labevent_id); - --- microbiologyevents - -ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_pk CASCADE; -ALTER TABLE mimiciv_hosp.microbiologyevents -ADD CONSTRAINT microbiologyevents_pk - PRIMARY KEY (microevent_id); - --- patients - -ALTER TABLE mimiciv_hosp.patients DROP CONSTRAINT IF EXISTS patients_pk CASCADE; -ALTER TABLE mimiciv_hosp.patients -ADD CONSTRAINT patients_pk - PRIMARY KEY (subject_id); - --- pharmacy - -ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_pk CASCADE; -ALTER TABLE mimiciv_hosp.pharmacy -ADD CONSTRAINT pharmacy_pk - PRIMARY KEY (pharmacy_id); - --- poe_detail - -ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_pk CASCADE; -ALTER TABLE mimiciv_hosp.poe_detail -ADD CONSTRAINT poe_detail_pk - PRIMARY KEY (poe_id, field_name); - --- poe - -ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_pk CASCADE; -ALTER TABLE mimiciv_hosp.poe -ADD CONSTRAINT poe_pk - PRIMARY KEY (poe_id); - --- prescriptions - -ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_pk CASCADE; -ALTER TABLE mimiciv_hosp.prescriptions -ADD CONSTRAINT prescriptions_pk - PRIMARY KEY (pharmacy_id, drug_type, drug); - --- procedures_icd - -ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_pk CASCADE; -ALTER TABLE mimiciv_hosp.procedures_icd -ADD CONSTRAINT procedures_icd_pk - PRIMARY KEY (hadm_id, seq_num, icd_code, icd_version); - --- services - -ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_pk CASCADE; -ALTER TABLE mimiciv_hosp.services -ADD CONSTRAINT services_pk - PRIMARY KEY (hadm_id, transfertime, curr_service); - ---------- --- icu -- ---------- - --- datetimeevents - -ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_pk CASCADE; -ALTER TABLE mimiciv_icu.datetimeevents -ADD CONSTRAINT datetimeevents_pk - PRIMARY KEY (stay_id, itemid, charttime); - --- d_items - -ALTER TABLE mimiciv_icu.d_items DROP CONSTRAINT IF EXISTS d_items_pk CASCADE; -ALTER TABLE mimiciv_icu.d_items -ADD CONSTRAINT d_items_pk - PRIMARY KEY (itemid); - --- icustays - -ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_pk CASCADE; -ALTER TABLE mimiciv_icu.icustays -ADD CONSTRAINT icustays_pk - PRIMARY KEY (stay_id); - --- inputevents - -ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_pk CASCADE; -ALTER TABLE mimiciv_icu.inputevents -ADD CONSTRAINT inputevents_pk - PRIMARY KEY (orderid, itemid); - --- outputevents - -ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_pk CASCADE; -ALTER TABLE mimiciv_icu.outputevents -ADD CONSTRAINT outputevents_pk - PRIMARY KEY (stay_id, charttime, itemid); - --- procedureevents - -ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_pk CASCADE; -ALTER TABLE mimiciv_icu.procedureevents -ADD CONSTRAINT procedureevents_pk - PRIMARY KEY (orderid); - ---------------------------- ---------------------------- --- Creating Foreign Keys -- ---------------------------- ---------------------------- - ----------- --- hosp -- ----------- - --- admissions - -ALTER TABLE mimiciv_hosp.admissions DROP CONSTRAINT IF EXISTS admissions_patients_fk; -ALTER TABLE mimiciv_hosp.admissions -ADD CONSTRAINT admissions_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - --- diagnoses_icd - -ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_patients_fk; -ALTER TABLE mimiciv_hosp.diagnoses_icd -ADD CONSTRAINT diagnoses_icd_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.diagnoses_icd DROP CONSTRAINT IF EXISTS diagnoses_icd_admissions_fk; -ALTER TABLE mimiciv_hosp.diagnoses_icd -ADD CONSTRAINT diagnoses_icd_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- drgcodes - -ALTER TABLE mimiciv_hosp.drgcodes DROP CONSTRAINT IF EXISTS drgcodes_patients_fk; -ALTER TABLE mimiciv_hosp.drgcodes -ADD CONSTRAINT drgcodes_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.drgcodes DROP CONSTRAINT IF EXISTS drgcodes_admissions_fk; -ALTER TABLE mimiciv_hosp.drgcodes -ADD CONSTRAINT drgcodes_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- emar_detail - -ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_patients_fk; -ALTER TABLE mimiciv_hosp.emar_detail -ADD CONSTRAINT emar_detail_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.emar_detail DROP CONSTRAINT IF EXISTS emar_detail_emar_fk; -ALTER TABLE mimiciv_hosp.emar_detail -ADD CONSTRAINT emar_detail_emar_fk - FOREIGN KEY (emar_id) - REFERENCES mimiciv_hosp.emar (emar_id); - --- emar - -ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_patients_fk; -ALTER TABLE mimiciv_hosp.emar -ADD CONSTRAINT emar_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.emar DROP CONSTRAINT IF EXISTS emar_admissions_fk; -ALTER TABLE mimiciv_hosp.emar -ADD CONSTRAINT emar_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- hcpcsevents - -ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_patients_fk; -ALTER TABLE mimiciv_hosp.hcpcsevents -ADD CONSTRAINT hcpcsevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_admissions_fk; -ALTER TABLE mimiciv_hosp.hcpcsevents -ADD CONSTRAINT hcpcsevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_hosp.hcpcsevents DROP CONSTRAINT IF EXISTS hcpcsevents_d_hcpcs_fk; -ALTER TABLE mimiciv_hosp.hcpcsevents -ADD CONSTRAINT hcpcsevents_d_hcpcs_fk - FOREIGN KEY (hcpcs_cd) - REFERENCES mimiciv_hosp.d_hcpcs (code); - --- labevents - -ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_patients_fk; -ALTER TABLE mimiciv_hosp.labevents -ADD CONSTRAINT labevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.labevents DROP CONSTRAINT IF EXISTS labevents_d_labitems_fk; -ALTER TABLE mimiciv_hosp.labevents -ADD CONSTRAINT labevents_d_labitems_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_hosp.d_labitems (itemid); - --- microbiologyevents - -ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_patients_fk; -ALTER TABLE mimiciv_hosp.microbiologyevents -ADD CONSTRAINT microbiologyevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.microbiologyevents DROP CONSTRAINT IF EXISTS microbiologyevents_admissions_fk; -ALTER TABLE mimiciv_hosp.microbiologyevents -ADD CONSTRAINT microbiologyevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- pharmacy - -ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_patients_fk; -ALTER TABLE mimiciv_hosp.pharmacy -ADD CONSTRAINT pharmacy_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.pharmacy DROP CONSTRAINT IF EXISTS pharmacy_admissions_fk; -ALTER TABLE mimiciv_hosp.pharmacy -ADD CONSTRAINT pharmacy_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- poe_detail - -ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_patients_fk; -ALTER TABLE mimiciv_hosp.poe_detail -ADD CONSTRAINT poe_detail_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.poe_detail DROP CONSTRAINT IF EXISTS poe_detail_poe_fk; -ALTER TABLE mimiciv_hosp.poe_detail -ADD CONSTRAINT poe_detail_poe_fk - FOREIGN KEY (poe_id) - REFERENCES mimiciv_hosp.poe (poe_id); - --- poe - -ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_patients_fk; -ALTER TABLE mimiciv_hosp.poe -ADD CONSTRAINT poe_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.poe DROP CONSTRAINT IF EXISTS poe_admissions_fk; -ALTER TABLE mimiciv_hosp.poe -ADD CONSTRAINT poe_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- prescriptions - -ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_patients_fk; -ALTER TABLE mimiciv_hosp.prescriptions -ADD CONSTRAINT prescriptions_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.prescriptions DROP CONSTRAINT IF EXISTS prescriptions_admissions_fk; -ALTER TABLE mimiciv_hosp.prescriptions -ADD CONSTRAINT prescriptions_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- procedures_icd - -ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_patients_fk; -ALTER TABLE mimiciv_hosp.procedures_icd -ADD CONSTRAINT procedures_icd_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.procedures_icd DROP CONSTRAINT IF EXISTS procedures_icd_admissions_fk; -ALTER TABLE mimiciv_hosp.procedures_icd -ADD CONSTRAINT procedures_icd_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- services - -ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_patients_fk; -ALTER TABLE mimiciv_hosp.services -ADD CONSTRAINT services_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_hosp.services DROP CONSTRAINT IF EXISTS services_admissions_fk; -ALTER TABLE mimiciv_hosp.services -ADD CONSTRAINT services_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- transfers - -ALTER TABLE mimiciv_hosp.transfers DROP CONSTRAINT IF EXISTS transfers_pk CASCADE; -ALTER TABLE mimiciv_hosp.transfers -ADD CONSTRAINT transfers_pk - PRIMARY KEY (transfer_id); - --- transfers - -ALTER TABLE mimiciv_hosp.transfers DROP CONSTRAINT IF EXISTS transfers_patients_fk; -ALTER TABLE mimiciv_hosp.transfers -ADD CONSTRAINT transfers_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - - ---------- --- icu -- ---------- - --- chartevents - -ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_patients_fk; -ALTER TABLE mimiciv_icu.chartevents -ADD CONSTRAINT chartevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_admissions_fk; -ALTER TABLE mimiciv_icu.chartevents -ADD CONSTRAINT chartevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_icustays_fk; -ALTER TABLE mimiciv_icu.chartevents -ADD CONSTRAINT chartevents_icustays_fk - FOREIGN KEY (stay_id) - REFERENCES mimiciv_icu.icustays (stay_id); - -ALTER TABLE mimiciv_icu.chartevents DROP CONSTRAINT IF EXISTS chartevents_d_items_fk; -ALTER TABLE mimiciv_icu.chartevents -ADD CONSTRAINT chartevents_d_items_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_icu.d_items (itemid); - --- datetimeevents - -ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_patients_fk; -ALTER TABLE mimiciv_icu.datetimeevents -ADD CONSTRAINT datetimeevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_admissions_fk; -ALTER TABLE mimiciv_icu.datetimeevents -ADD CONSTRAINT datetimeevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_icustays_fk; -ALTER TABLE mimiciv_icu.datetimeevents -ADD CONSTRAINT datetimeevents_icustays_fk - FOREIGN KEY (stay_id) - REFERENCES mimiciv_icu.icustays (stay_id); - -ALTER TABLE mimiciv_icu.datetimeevents DROP CONSTRAINT IF EXISTS datetimeevents_d_items_fk; -ALTER TABLE mimiciv_icu.datetimeevents -ADD CONSTRAINT datetimeevents_d_items_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_icu.d_items (itemid); - --- icustays - -ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_patients_fk; -ALTER TABLE mimiciv_icu.icustays -ADD CONSTRAINT icustays_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.icustays DROP CONSTRAINT IF EXISTS icustays_admissions_fk; -ALTER TABLE mimiciv_icu.icustays -ADD CONSTRAINT icustays_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - --- inputevents - -ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_patients_fk; -ALTER TABLE mimiciv_icu.inputevents -ADD CONSTRAINT inputevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_admissions_fk; -ALTER TABLE mimiciv_icu.inputevents -ADD CONSTRAINT inputevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_icustays_fk; -ALTER TABLE mimiciv_icu.inputevents -ADD CONSTRAINT inputevents_icustays_fk - FOREIGN KEY (stay_id) - REFERENCES mimiciv_icu.icustays (stay_id); - -ALTER TABLE mimiciv_icu.inputevents DROP CONSTRAINT IF EXISTS inputevents_d_items_fk; -ALTER TABLE mimiciv_icu.inputevents -ADD CONSTRAINT inputevents_d_items_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_icu.d_items (itemid); - --- outputevents - -ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_patients_fk; -ALTER TABLE mimiciv_icu.outputevents -ADD CONSTRAINT outputevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_admissions_fk; -ALTER TABLE mimiciv_icu.outputevents -ADD CONSTRAINT outputevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_icustays_fk; -ALTER TABLE mimiciv_icu.outputevents -ADD CONSTRAINT outputevents_icustays_fk - FOREIGN KEY (stay_id) - REFERENCES mimiciv_icu.icustays (stay_id); - -ALTER TABLE mimiciv_icu.outputevents DROP CONSTRAINT IF EXISTS outputevents_d_items_fk; -ALTER TABLE mimiciv_icu.outputevents -ADD CONSTRAINT outputevents_d_items_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_icu.d_items (itemid); - --- procedureevents - -ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_patients_fk; -ALTER TABLE mimiciv_icu.procedureevents -ADD CONSTRAINT procedureevents_patients_fk - FOREIGN KEY (subject_id) - REFERENCES mimiciv_hosp.patients (subject_id); - -ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_admissions_fk; -ALTER TABLE mimiciv_icu.procedureevents -ADD CONSTRAINT procedureevents_admissions_fk - FOREIGN KEY (hadm_id) - REFERENCES mimiciv_hosp.admissions (hadm_id); - -ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_icustays_fk; -ALTER TABLE mimiciv_icu.procedureevents -ADD CONSTRAINT procedureevents_icustays_fk - FOREIGN KEY (stay_id) - REFERENCES mimiciv_icu.icustays (stay_id); - -ALTER TABLE mimiciv_icu.procedureevents DROP CONSTRAINT IF EXISTS procedureevents_d_items_fk; -ALTER TABLE mimiciv_icu.procedureevents -ADD CONSTRAINT procedureevents_d_items_fk - FOREIGN KEY (itemid) - REFERENCES mimiciv_icu.d_items (itemid); diff --git a/mimic-iv/buildmimic/postgres/docker/src/04_index.sql b/mimic-iv/buildmimic/postgres/docker/src/04_index.sql deleted file mode 100644 index 948a0d4c0..000000000 --- a/mimic-iv/buildmimic/postgres/docker/src/04_index.sql +++ /dev/null @@ -1,232 +0,0 @@ --------------------------------------- --------------------------------------- --- Indexes for all MIMIC-IV modules -- --------------------------------------- --------------------------------------- - ----------- --- hosp -- ----------- - -SET search_path TO mimiciv_hosp; - --- admissions - -DROP INDEX IF EXISTS admissions_idx01; -CREATE INDEX admissions_idx01 - ON admissions (admittime, dischtime, deathtime); - --- d_icd_diagnoses - -DROP INDEX IF EXISTS D_ICD_DIAG_idx02; -CREATE INDEX D_ICD_DIAG_idx02 - ON D_ICD_DIAGNOSES (LONG_TITLE); - --- D_ICD_PROCEDURES - -DROP INDEX IF EXISTS D_ICD_PROC_idx02; -CREATE INDEX D_ICD_PROC_idx02 - ON D_ICD_PROCEDURES (LONG_TITLE); - --- drgcodes - -DROP INDEX IF EXISTS drgcodes_idx01; -CREATE INDEX drgcodes_idx01 - ON drgcodes (drg_code, drg_type); - -DROP INDEX IF EXISTS drgcodes_idx02; -CREATE INDEX drgcodes_idx02 - ON drgcodes (description, drg_severity); - --- d_labitems - -DROP INDEX IF EXISTS d_labitems_idx01; -CREATE INDEX d_labitems_idx01 - ON d_labitems (label, fluid, category); - --- emar_detail - -DROP INDEX IF EXISTS emar_detail_idx01; -CREATE INDEX emar_detail_idx01 - ON emar_detail (pharmacy_id); - -DROP INDEX IF EXISTS emar_detail_idx02; -CREATE INDEX emar_detail_idx02 - ON emar_detail (product_code); - -DROP INDEX IF EXISTS emar_detail_idx03; -CREATE INDEX emar_detail_idx03 - ON emar_detail (route, site, side); - -DROP INDEX IF EXISTS EMAR_DET_idx04; -CREATE INDEX EMAR_DET_idx04 - ON EMAR_DETAIL (PRODUCT_DESCRIPTION); - --- emar - -DROP INDEX IF EXISTS emar_idx01; -CREATE INDEX emar_idx01 - ON emar (poe_id); - -DROP INDEX IF EXISTS emar_idx02; -CREATE INDEX emar_idx02 - ON emar (pharmacy_id); - -DROP INDEX IF EXISTS emar_idx03; -CREATE INDEX emar_idx03 - ON emar (charttime, scheduletime, storetime); - -DROP INDEX IF EXISTS emar_idx04; -CREATE INDEX emar_idx04 - ON emar (medication); - --- HCPCSEVENTS - -DROP INDEX IF EXISTS HCPCSEVENTS_idx04; -CREATE INDEX HCPCSEVENTS_idx04 - ON HCPCSEVENTS (SHORT_DESCRIPTION); - --- labevents - -DROP INDEX IF EXISTS labevents_idx01; -CREATE INDEX labevents_idx01 - ON labevents (charttime, storetime); - -DROP INDEX IF EXISTS labevents_idx02; -CREATE INDEX labevents_idx02 - ON labevents (specimen_id); - --- microbiologyevents - -DROP INDEX IF EXISTS microbiologyevents_idx01; -CREATE INDEX microbiologyevents_idx01 - ON microbiologyevents (chartdate, charttime, storedate, storetime); - -DROP INDEX IF EXISTS microbiologyevents_idx02; -CREATE INDEX microbiologyevents_idx02 - ON microbiologyevents (spec_itemid, test_itemid, org_itemid, ab_itemid); - -DROP INDEX IF EXISTS microbiologyevents_idx03; -CREATE INDEX microbiologyevents_idx03 - ON microbiologyevents (micro_specimen_id); - --- patients -DROP INDEX IF EXISTS patients_idx01; -CREATE INDEX patients_idx01 - ON patients (anchor_age); - -DROP INDEX IF EXISTS patients_idx02; -CREATE INDEX patients_idx02 - ON patients (anchor_year); - --- pharmacy - -DROP INDEX IF EXISTS pharmacy_idx01; -CREATE INDEX pharmacy_idx01 - ON pharmacy (poe_id); - -DROP INDEX IF EXISTS pharmacy_idx02; -CREATE INDEX pharmacy_idx02 - ON pharmacy (starttime, stoptime); - -DROP INDEX IF EXISTS pharmacy_idx03; -CREATE INDEX pharmacy_idx03 - ON pharmacy (medication); - -DROP INDEX IF EXISTS pharmacy_idx04; -CREATE INDEX pharmacy_idx04 - ON pharmacy (route); - --- poe - -DROP INDEX IF EXISTS poe_idx01; -CREATE INDEX poe_idx01 - ON poe (order_type); - --- prescriptions - -DROP INDEX IF EXISTS prescriptions_idx01; -CREATE INDEX prescriptions_idx01 - ON prescriptions (starttime, stoptime); - --- transfers - -DROP INDEX IF EXISTS transfers_idx01; -CREATE INDEX transfers_idx01 - ON transfers (hadm_id); - -DROP INDEX IF EXISTS transfers_idx02; -CREATE INDEX transfers_idx02 - ON transfers (intime); - -DROP INDEX IF EXISTS transfers_idx03; -CREATE INDEX transfers_idx03 - ON transfers (careunit); - ---------- --- icu -- ---------- - -SET search_path TO mimiciv_icu; - --- chartevents - -DROP INDEX IF EXISTS chartevents_idx01; -CREATE INDEX chartevents_idx01 - ON chartevents (charttime, storetime); - --- datetimeevents - -DROP INDEX IF EXISTS datetimeevents_idx01; -CREATE INDEX datetimeevents_idx01 - ON datetimeevents (charttime, storetime); - -DROP INDEX IF EXISTS datetimeevents_idx02; -CREATE INDEX datetimeevents_idx02 - ON datetimeevents (value); - --- d_items - -DROP INDEX IF EXISTS d_items_idx01; -CREATE INDEX d_items_idx01 - ON d_items (label, abbreviation); - -DROP INDEX IF EXISTS d_items_idx02; -CREATE INDEX d_items_idx02 - ON d_items (category); - --- icustays - -DROP INDEX IF EXISTS icustays_idx01; -CREATE INDEX icustays_idx01 - ON icustays (first_careunit, last_careunit); - -DROP INDEX IF EXISTS icustays_idx02; -CREATE INDEX icustays_idx02 - ON icustays (intime, outtime); - --- inputevents - -DROP INDEX IF EXISTS inputevents_idx01; -CREATE INDEX inputevents_idx01 - ON inputevents (starttime, endtime); - -DROP INDEX IF EXISTS inputevents_idx02; -CREATE INDEX inputevents_idx02 - ON inputevents (ordercategorydescription); - --- outputevents - -DROP INDEX IF EXISTS outputevents_idx01; -CREATE INDEX outputevents_idx01 - ON outputevents (charttime, storetime); - --- procedureevents - -DROP INDEX IF EXISTS procedureevents_idx01; -CREATE INDEX procedureevents_idx01 - ON procedureevents (starttime, endtime); - -DROP INDEX IF EXISTS procedureevents_idx02; -CREATE INDEX procedureevents_idx02 - ON procedureevents (ordercategoryname); From 4ab8483628ce00d2b553ebb907a014946f38c5fd Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 10 Aug 2026 19:39:08 -0400 Subject: [PATCH 5/8] update to use build_mimic.sh convenience script --- .github/actions/load-mimic-psql/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/load-mimic-psql/action.yml b/.github/actions/load-mimic-psql/action.yml index 6b91b8ca5..03f9bebae 100644 --- a/.github/actions/load-mimic-psql/action.yml +++ b/.github/actions/load-mimic-psql/action.yml @@ -1,5 +1,5 @@ name: "Load MIMIC-IV demo into PostgreSQL" -description: "Creates the MIMIC-IV schemas and loads the demo hosp/icu data. Requires psql on PATH and PG* connection env vars." +description: "Builds the MIMIC-IV schemas, data, constraints and indexes from the demo hosp/icu data using build_mimic.sh. Requires psql on PATH and PG* connection env vars." inputs: mimic_data_dir: @@ -15,9 +15,11 @@ runs: # Resolve the data dir at shell time so $GITHUB_WORKSPACE points at the # container-mounted path. The github.workspace context yields the host # path, which does not exist inside a container job. + env: + # Load only - we validate/build concepts elsewhere. + MIMIC_MAKE_CONCEPTS: "false" + MIMIC_VALIDATE: "false" run: | data_dir="${{ inputs.mimic_data_dir }}" data_dir="${data_dir:-$GITHUB_WORKSPACE}" - psql -q -v ON_ERROR_STOP=1 -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/create.sql" - psql -q -v ON_ERROR_STOP=1 -v mimic_data_dir="$data_dir" \ - -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/load_gz.sql" + "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/build_mimic.sh" "$data_dir" From 4423034d7661880e298b24166381046f7a79956d Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 10 Aug 2026 19:39:22 -0400 Subject: [PATCH 6/8] add CI to test docker builds of psql/duckdb --- .github/workflows/docker-build.yml | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/docker-build.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..a70e66c2e --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,89 @@ +# Build and run the containerized MIMIC-IV builds against the demo dataset. +name: docker build + +on: + pull_request: + paths: + - 'mimic-iv/buildmimic/postgres/**' + - 'mimic-iv/buildmimic/duckdb/**' + - 'mimic-iv/buildmimic/download_data.sh' + - '.github/workflows/docker-build.yml' + push: + branches: + - main + paths: + - 'mimic-iv/buildmimic/postgres/**' + - 'mimic-iv/buildmimic/duckdb/**' + - '.github/workflows/docker-build.yml' + +# Cancel superseded runs when a PR is pushed again. +concurrency: + group: docker-build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + defaults: + run: + # Name bash explicitly to get `-eo pipefail`. + shell: bash + + env: + DUCKDB_DOCKER: mimic-iv/buildmimic/duckdb/docker + POSTGRES_DOCKER: mimic-iv/buildmimic/postgres/docker + MIMIC_DATA_DIR: ${{ github.workspace }} + MIMIC_OUTPUT_DIR: ${{ github.workspace }}/duckdb-out + + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/download-demo + - uses: ./.github/actions/setup-duckdb + + - name: Build both images + run: | + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" build + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" build + + # --exit-code-from implies --abort-on-container-exit and returns the build + # job's own status, so a failed build fails the step. + - name: Build MIMIC-IV in DuckDB + run: | + mkdir -p "$MIMIC_OUTPUT_DIR" + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" \ + up --exit-code-from mimic-build | tee duckdb.log + if grep -F -q "FAILED" duckdb.log; then + echo "::error::DuckDB container row-count validation failed:" + grep -F "FAILED" duckdb.log + exit 1 + fi + + - name: Read the built database with the runner's own duckdb + run: | + rows=$(duckdb "$MIMIC_OUTPUT_DIR/mimic4.db" -noheader -list \ + -c "SELECT count(*) FROM mimiciv_derived.sepsis3") + echo "mimiciv_derived.sepsis3: ${rows} rows" + if [ -z "${rows}" ] || [ "${rows}" -eq 0 ]; then + echo "::error::The concepts are missing from the container-built database." + exit 1 + fi + + - name: Build MIMIC-IV in PostgreSQL + run: | + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" \ + up --exit-code-from mimic-build | tee psql.log + if grep -F -q "FAILED" psql.log; then + echo "::error::PostgreSQL container row-count validation failed:" + grep -F "FAILED" psql.log + exit 1 + fi + + - name: Tear down + if: always() + run: | + docker compose -f "$DUCKDB_DOCKER/docker-compose.yml" down -v || true + docker compose -f "$POSTGRES_DOCKER/docker-compose.yml" down -v || true From 46cf3def1503d7815b1e9ec9a8ff3d0568c60def Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 10 Aug 2026 21:18:09 -0400 Subject: [PATCH 7/8] name both docker compose jobs to avoid conflicts in CI --- .../bigquery/schemas/demo_subject_id_schema.json | 7 +++++++ mimic-iv/buildmimic/duckdb/docker/docker-compose.yml | 2 ++ mimic-iv/buildmimic/postgres/docker/docker-compose.yml | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json diff --git a/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json b/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json new file mode 100644 index 000000000..a19efcd57 --- /dev/null +++ b/mimic-iv/buildmimic/bigquery/schemas/demo_subject_id_schema.json @@ -0,0 +1,7 @@ +[ + { + "name": "subject_id", + "type": "INT64", + "mode": "REQUIRED" + } +] \ No newline at end of file diff --git a/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml index 6dec5c7ba..2bf9d9990 100644 --- a/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml +++ b/mimic-iv/buildmimic/duckdb/docker/docker-compose.yml @@ -1,3 +1,5 @@ +name: mimic-duckdb + services: # Loads the data and derives the concepts, then exits. # Re-running it resumes an interrupted build. diff --git a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml index 8da8bbfcf..4fdfa3e24 100644 --- a/mimic-iv/buildmimic/postgres/docker/docker-compose.yml +++ b/mimic-iv/buildmimic/postgres/docker/docker-compose.yml @@ -1,3 +1,5 @@ +name: mimic-postgres + services: mimic-db: container_name: mimic-db From a8c8bcf769c12b163a5e3114b7c07ba6cb4109da Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 10 Aug 2026 21:37:24 -0400 Subject: [PATCH 8/8] use bind mount to infer user ID for file ownership --- mimic-iv/buildmimic/duckdb/docker/Dockerfile | 3 ++- .../buildmimic/duckdb/docker/Dockerfile.dockerignore | 1 + mimic-iv/buildmimic/duckdb/docker/README.md | 4 ---- .../buildmimic/duckdb/docker/docker-entrypoint.sh | 11 +++++++++++ 4 files changed, 14 insertions(+), 5 deletions(-) create mode 100755 mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile b/mimic-iv/buildmimic/duckdb/docker/Dockerfile index ded652d0a..95bfc9c9c 100644 --- a/mimic-iv/buildmimic/duckdb/docker/Dockerfile +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update \ && duckdb --version COPY buildmimic/duckdb/build_mimic.sh /mimic/buildmimic/duckdb/ +COPY buildmimic/duckdb/docker/docker-entrypoint.sh /mimic/ # create.sql, validate.sql and validate_demo.sql are shared with the postgres # build; build_mimic.sh reads them from ../postgres. COPY buildmimic/postgres/create.sql buildmimic/postgres/validate*.sql /mimic/buildmimic/postgres/ @@ -36,5 +37,5 @@ COPY concepts_duckdb/ /mimic/concepts_duckdb/ # The database is written here. WORKDIR /out -ENTRYPOINT ["/mimic/buildmimic/duckdb/build_mimic.sh"] +ENTRYPOINT ["/mimic/docker-entrypoint.sh"] CMD ["/data"] diff --git a/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore index badd2c070..701d3b5cc 100644 --- a/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore +++ b/mimic-iv/buildmimic/duckdb/docker/Dockerfile.dockerignore @@ -2,6 +2,7 @@ # Exclude everything, then add back only what the image needs. * !buildmimic/duckdb/build_mimic.sh +!buildmimic/duckdb/docker/docker-entrypoint.sh !buildmimic/postgres/create.sql !buildmimic/postgres/validate.sql !buildmimic/postgres/validate_demo.sql diff --git a/mimic-iv/buildmimic/duckdb/docker/README.md b/mimic-iv/buildmimic/duckdb/docker/README.md index 6ba50bd62..527ed74ff 100644 --- a/mimic-iv/buildmimic/duckdb/docker/README.md +++ b/mimic-iv/buildmimic/duckdb/docker/README.md @@ -101,7 +101,3 @@ duckdb ./mimic-db/mimic4.db # otherwise, from inside a throwaway container using the pinned version (not recommended) docker compose run --rm --entrypoint duckdb mimic-build /out/mimic4.db ``` - -On Linux the file is written as `root`, since that is what the container runs -as. `sudo chown "$USER" ./mimic-db/mimic4.db` if you would rather own it. Docker -Desktop on macOS and Windows maps ownership to you automatically. diff --git a/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh b/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh new file mode 100755 index 000000000..ee96c0ba7 --- /dev/null +++ b/mimic-iv/buildmimic/duckdb/docker/docker-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# This entrypoint exists as we need to infer the user ID from the bind mount to +# re-assign ownership of the output db file from root to the user. +set -eu + +give_output_to_host_user() { + chown -R --reference=/out /out 2>/dev/null || true +} +trap give_output_to_host_user EXIT + +/mimic/buildmimic/duckdb/build_mimic.sh "$@"