|
| 1 | +#!/usr/bin/env bash |
| 2 | +# discoverResiduals.sh |
| 3 | +# |
| 4 | +# Runs locally. For each run in a batch file, calls makeResidualLists.sh |
| 5 | +# with --alien to query AliEn for o2tpc_residuals*.root files and build |
| 6 | +# per-period residual lists under lists/<RUN>/. |
| 7 | +# |
| 8 | +# Runs are processed in parallel using GNU parallel. |
| 9 | +# |
| 10 | +# Usage: |
| 11 | +# ./discoverResiduals.sh <batch_file> [--calib-pass <pass>] |
| 12 | +# |
| 13 | +# Batch file format (one run per line, whitespace-separated): |
| 14 | +# <period_token> <run> |
| 15 | +# e.g.: LHC26ak 572557 |
| 16 | +# |
| 17 | +# PERIOD is derived from the first field by stripping from the first '.' onward |
| 18 | +# (e.g. LHC26ab.b5p → LHC26ab — a trailing '.xyz' suffix is tolerated but not |
| 19 | +# required). |
| 20 | +# YEAR is derived from the two-digit year in the period token |
| 21 | +# (e.g. LHC26ab → 2026). |
| 22 | +# |
| 23 | +# Dependencies: alien.py, parallel, makeResidualLists.sh |
| 24 | + |
| 25 | +set -euo pipefail |
| 26 | + |
| 27 | +# --------------------------------------------------------------------------- |
| 28 | +# Parse arguments |
| 29 | +# --------------------------------------------------------------------------- |
| 30 | +BATCH_FILE="" |
| 31 | +CALIB_PASS="cpass2_residuals" |
| 32 | + |
| 33 | +while [[ $# -gt 0 ]]; do |
| 34 | + case "$1" in |
| 35 | + --calib-pass) |
| 36 | + [[ $# -lt 2 ]] && { echo "ERROR: --calib-pass requires an argument"; exit 1; } |
| 37 | + CALIB_PASS="$2"; shift 2 ;; |
| 38 | + -*) |
| 39 | + echo "ERROR: Unknown option: $1"; exit 1 ;; |
| 40 | + *) |
| 41 | + [[ -n "${BATCH_FILE}" ]] && { echo "ERROR: unexpected extra argument: $1"; exit 1; } |
| 42 | + BATCH_FILE="$1"; shift ;; |
| 43 | + esac |
| 44 | +done |
| 45 | + |
| 46 | +[[ -z "${BATCH_FILE}" ]] && { echo "Usage: $0 <batch_file> [--calib-pass <pass>]"; exit 1; } |
| 47 | +[[ -f "${BATCH_FILE}" ]] || { echo "ERROR: batch file not found: ${BATCH_FILE}"; exit 1; } |
| 48 | + |
| 49 | +# --------------------------------------------------------------------------- |
| 50 | +# Parse the batch file into one "<period_token> <run>" entry per line |
| 51 | +# --------------------------------------------------------------------------- |
| 52 | +# The period is resolved PER LINE, not once from the first line: the batch format carries a period |
| 53 | +# token on every row, so a batch legitimately spanning two periods must not silently inherit the |
| 54 | +# first row's period for all of its runs (that produced a valid-looking /alice/data/<year>/<wrong |
| 55 | +# period>/<run>/ query, which just finds nothing and degrades into a per-run warning). |
| 56 | +mapfile -t ENTRIES < <(awk 'NF > 0 && !/^[[:space:]]*#/ {print $1, $2}' "${BATCH_FILE}") |
| 57 | +[[ ${#ENTRIES[@]} -gt 0 ]] || { echo "ERROR: no runs found in ${BATCH_FILE}"; exit 1; } |
| 58 | + |
| 59 | +# Validate every entry up front -- a malformed row should stop the batch here, not surface as one |
| 60 | +# failed run out of fifty after the parallel fan-out has already started. |
| 61 | +RUNS=() |
| 62 | +PERIODS_SEEN=() |
| 63 | +for entry in "${ENTRIES[@]}"; do |
| 64 | + read -r _tok _run <<< "${entry}" |
| 65 | + [[ -n "${_tok}" && -n "${_run}" ]] \ |
| 66 | + || { echo "ERROR: malformed line in ${BATCH_FILE} (need '<period_token> <run>'): '${entry}'"; exit 1; } |
| 67 | + [[ "${_run}" =~ ^[0-9]+$ ]] \ |
| 68 | + || { echo "ERROR: run number is not numeric in ${BATCH_FILE}: '${entry}'"; exit 1; } |
| 69 | + # Field 1 example: LHC26ab.b5p → PERIOD=LHC26ab, YEAR=2026 |
| 70 | + [[ "${_tok%%.*}" =~ LHC([0-9]{2}) ]] \ |
| 71 | + || { echo "ERROR: cannot extract year from period token '${_tok}' (line: '${entry}')"; exit 1; } |
| 72 | + RUNS+=("${_run}") |
| 73 | + PERIODS_SEEN+=("${_tok%%.*}") |
| 74 | +done |
| 75 | + |
| 76 | +mapfile -t PERIODS_UNIQ < <(printf '%s\n' "${PERIODS_SEEN[@]}" | sort -u) |
| 77 | + |
| 78 | +# --------------------------------------------------------------------------- |
| 79 | +# Fixed configuration |
| 80 | +# --------------------------------------------------------------------------- |
| 81 | +# Maximum number of runs to process in parallel (0 = one per CPU core) |
| 82 | +N_PARALLEL=8 |
| 83 | + |
| 84 | +# Root of the output tree (relative or absolute) |
| 85 | +WORK_DIR="./lists" |
| 86 | + |
| 87 | +# Directory containing this script |
| 88 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 89 | + |
| 90 | +# Path to makeResidualLists.sh (co-located here) |
| 91 | +MAKE_LISTS_SCRIPT="${SCRIPT_DIR}/makeResidualLists.sh" |
| 92 | + |
| 93 | +# --------------------------------------------------------------------------- |
| 94 | + |
| 95 | +echo "Batch file : ${BATCH_FILE}" |
| 96 | +echo "Period(s) : ${PERIODS_UNIQ[*]}" |
| 97 | +echo "Calib pass : ${CALIB_PASS}" |
| 98 | +echo "Runs (${#RUNS[@]}): ${RUNS[*]}" |
| 99 | +echo |
| 100 | + |
| 101 | +mkdir -p "${WORK_DIR}" |
| 102 | + |
| 103 | +# --------------------------------------------------------------------------- |
| 104 | +# per_run — all work for a single run, called by parallel |
| 105 | +# |
| 106 | +# On success writes <RUN_DIR>/run_completed.txt containing the absolute |
| 107 | +# path of RUN_DIR so the parent shell can collect completed runs after |
| 108 | +# parallel finishes. |
| 109 | +# --------------------------------------------------------------------------- |
| 110 | +per_run() { |
| 111 | + local PERIOD_TOKEN="$1" # field 1 of this batch-file row |
| 112 | + local RUN="$2" # field 2 of this batch-file row |
| 113 | + local CALIB_PASS="$3" |
| 114 | + local WORK_DIR="$4" |
| 115 | + local MAKE_LISTS_SCRIPT="$5" |
| 116 | + |
| 117 | + # Period/year come from THIS row, so a multi-period batch resolves each run against its own period. |
| 118 | + # Already validated in the caller's pre-flight loop, so a parse failure here would be a bug. |
| 119 | + local PERIOD YEAR |
| 120 | + PERIOD="${PERIOD_TOKEN%%.*}" # strip suffix after first '.' |
| 121 | + [[ "${PERIOD}" =~ LHC([0-9]{2}) ]] |
| 122 | + YEAR="20${BASH_REMATCH[1]}" |
| 123 | + |
| 124 | + local RUN_DIR="${WORK_DIR}/${RUN}" |
| 125 | + mkdir -p "${RUN_DIR}" |
| 126 | + |
| 127 | + local ALIEN_PATH="/alice/data/${YEAR}/${PERIOD}/${RUN}/${CALIB_PASS}/" |
| 128 | + |
| 129 | + echo "========================================" |
| 130 | + echo "Processing run ${RUN} (period ${PERIOD}, year ${YEAR})" |
| 131 | + echo "========================================" |
| 132 | + |
| 133 | + echo " [${RUN}] Calling makeResidualLists.sh (alien path: ${ALIEN_PATH})..." |
| 134 | + ( |
| 135 | + cd "${RUN_DIR}" |
| 136 | + run="${RUN}" "${MAKE_LISTS_SCRIPT}" --subdir "${CALIB_PASS}" --alien --alien-path "${ALIEN_PATH}" |
| 137 | + ) || { |
| 138 | + echo " [${RUN}] WARNING: makeResidualLists.sh failed, skipping." |
| 139 | + return 0 |
| 140 | + } |
| 141 | + |
| 142 | + echo " [${RUN}] Done." |
| 143 | + |
| 144 | + # Signal success to the parent by writing the absolute run dir path |
| 145 | + realpath "${RUN_DIR}" > "${RUN_DIR}/run_completed.txt" |
| 146 | +} |
| 147 | + |
| 148 | +# Export everything parallel needs to call per_run in a subshell |
| 149 | +export -f per_run |
| 150 | + |
| 151 | +# --------------------------------------------------------------------------- |
| 152 | +# Run all runs in parallel |
| 153 | +# --------------------------------------------------------------------------- |
| 154 | +echo "========================================" |
| 155 | +echo "Launching ${#RUNS[@]} runs with N_PARALLEL=${N_PARALLEL}..." |
| 156 | +echo "========================================" |
| 157 | + |
| 158 | +# Fan out over the full "<period_token> <run>" entries, not bare run numbers, so each job carries its |
| 159 | +# own period. --colsep splits each entry into {1}=period token and {2}=run; note that under --colsep |
| 160 | +# a bare {} means column 1 ONLY, so both columns must be passed explicitly. --tagstring uses {2} to |
| 161 | +# keep the log prefix to just the run number. |
| 162 | +# shellcheck disable=SC1083 # {1}/{2} are GNU parallel replacement strings, not brace expansions |
| 163 | +parallel \ |
| 164 | + --jobs "${N_PARALLEL}" \ |
| 165 | + --line-buffer \ |
| 166 | + --colsep ' ' \ |
| 167 | + --tagstring "[{2}]" \ |
| 168 | + per_run {1} {2} "${CALIB_PASS}" "${WORK_DIR}" "${MAKE_LISTS_SCRIPT}" \ |
| 169 | + ::: "${ENTRIES[@]}" |
| 170 | + |
| 171 | +# --------------------------------------------------------------------------- |
| 172 | +# Collect completed runs (sentinel files written by per_run on success) |
| 173 | +# --------------------------------------------------------------------------- |
| 174 | +COMPLETED_RUNS=() |
| 175 | +for RUN in "${RUNS[@]}"; do |
| 176 | + SENTINEL="${WORK_DIR}/${RUN}/run_completed.txt" |
| 177 | + if [[ -f "${SENTINEL}" ]]; then |
| 178 | + COMPLETED_RUNS+=("$(cat "${SENTINEL}")") |
| 179 | + rm -f "${SENTINEL}" |
| 180 | + fi |
| 181 | +done |
| 182 | + |
| 183 | +echo "========================================" |
| 184 | +echo "All runs processed. ${#COMPLETED_RUNS[@]}/${#RUNS[@]} succeeded." |
| 185 | +echo "========================================" |
| 186 | + |
| 187 | +if (( ${#COMPLETED_RUNS[@]} == 0 )); then |
| 188 | + echo "No runs completed successfully." |
| 189 | + exit 0 |
| 190 | +fi |
| 191 | + |
| 192 | +# Sort for deterministic output order. Must come AFTER the empty check: expanding "${arr[@]}" on an |
| 193 | +# empty array is an "unbound variable" error under `set -u` in bash <= 4.3. |
| 194 | +IFS=$'\n' COMPLETED_RUNS=($(printf '%s\n' "${COMPLETED_RUNS[@]}" | sort)) |
| 195 | +unset IFS |
| 196 | + |
| 197 | +echo "Per-run slot-file indexes (input for stageSlotsForGrid.py):" |
| 198 | +for RUN_DIR in "${COMPLETED_RUNS[@]}"; do |
| 199 | + RUN="$(basename "${RUN_DIR}")" |
| 200 | + echo " ${RUN_DIR}/${RUN}.residuals_lists.txt" |
| 201 | +done |
0 commit comments