From 6de41eb700722198a6ed8b98b927c9bbab85b4be Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 12:13:00 -0400 Subject: [PATCH 1/2] Let the training dialogs email progress reports through viame monitor The desktop and web training dialogs gain an optional address field. It is passed to viame train as --monitor-email, which starts VIAME's training monitor next to the run to email start, error, epoch and completion reports. Mail delivery is configured on the training machine through VIAME_SMTP_SERVER and friends, so DIVE only carries the recipient. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- client/dive-common/apispec.ts | 2 + client/dive-common/constants.ts | 6 +++ .../platform/desktop/backend/native/viame.ts | 8 ++++ client/platform/desktop/constants.ts | 2 + client/platform/desktop/frontend/api.ts | 2 + .../frontend/components/MultiTrainingMenu.vue | 26 +++++++++++- client/platform/web-girder/api/rpc.service.ts | 5 ++- .../web-girder/views/RunTrainingMenu.vue | 40 ++++++++++++------- docs/Pipeline-Documentation.md | 6 +++ server/dive_server/crud_rpc.py | 2 + server/dive_tasks/run_training.py | 7 ++++ server/dive_utils/types.py | 1 + 12 files changed, 91 insertions(+), 16 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d4eed0dcd..e9af49d40 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -454,6 +454,8 @@ interface Api { path?: string; folderId?: string; }, + /** Address that receives progress and completion reports from the run. */ + monitorEmail?: string, ): Promise; /** diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index d949b067a..3545700e2 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -242,6 +242,11 @@ function simplifyTrainingName(item: string) { return item.replace('.conf', ''); } +/** Loose check for a single mailbox address, enough to catch typos in forms. */ +function isValidEmail(address: string) { + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(address.trim()); +} + export { DefaultVideoFPS, ImageSequenceType, @@ -284,4 +289,5 @@ export { pipelineCreatesDatasetMarkers, JsonConfigRegEx, simplifyTrainingName, + isValidEmail, }; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index a475c68d3..206c8bd04 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -956,6 +956,14 @@ async function train( command.push(labelsPath); } + // VIAME starts its training monitor next to the run, which emails progress + // and completion reports. Mail delivery is configured on the machine + // through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / VIAME_SMTP_PASSWORD). + if (runTrainingArgs.monitorEmail) { + command.push('--monitor-email'); + command.push(`"${runTrainingArgs.monitorEmail}"`); + } + const job = observeChild(spawn(command.join(' '), { shell: viameConstants.shell, cwd: jobWorkDir, diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 53caafd73..5ae8d1f69 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -234,6 +234,8 @@ export interface RunTraining extends JobArgs { }; // working directory of a prior interrupted run to continue from resumeWorkingDir?: string; + // address that receives progress and completion reports from the run + monitorEmail?: string; } export interface RunScoring extends JobArgs, ScoringJobArgs { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 825633d6d..124bf93d9 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -270,6 +270,7 @@ async function runTraining( path?: string; folderId?: string; }, + monitorEmail?: string, ): Promise { const args: RunTraining = { type: JobType.RunTraining, @@ -279,6 +280,7 @@ async function runTraining( annotatedFramesOnly, labelText, fineTuneModel, + monitorEmail, }; gpuJobQueue.enqueue(args); } diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index cf38b2cac..ae4aa7370 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -15,7 +15,7 @@ import { DatasetConfig, Pipelines, TrainingConfigs, useApi, Pipe, } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; -import { itemsPerPageOptions, simplifyTrainingName } from 'dive-common/constants'; +import { itemsPerPageOptions, simplifyTrainingName, isValidEmail } from 'dive-common/constants'; import { clientSettings } from 'dive-common/store/settings'; import { useRoute, useRouter } from 'vue-router/composables'; @@ -126,6 +126,9 @@ export default defineComponent({ const nameRules = [ (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'), ]; + const emailRules = [ + (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'), + ]; const data = reactive({ stagedItems: {} as Record, @@ -141,6 +144,7 @@ export default defineComponent({ models: {}, } as TrainingConfigs, annotatedFramesOnly: false, + monitorEmail: '', }); const headersTmpl: DataTableHeader[] = [ @@ -210,6 +214,7 @@ export default defineComponent({ stagedItems.value.length > 0 && data.selectedTrainingConfig && data.trainingOutputName + && (!data.monitorEmail || isValidEmail(data.monitorEmail)) )); async function deleteModel(item: Pipe) { @@ -285,6 +290,7 @@ export default defineComponent({ data.annotatedFramesOnly, labelText.value || undefined, foundTrainingModel, + data.monitorEmail.trim() || undefined, ); router.push({ name: 'jobs' }); } catch (err) { @@ -352,6 +358,7 @@ export default defineComponent({ resumeJob, discardJob, nameRules, + emailRules, itemsPerPageOptions, clientSettings, modelNames, @@ -533,6 +540,23 @@ export default defineComponent({ label="Fine Tune Model" /> + + + + +
diff --git a/client/platform/web-girder/api/rpc.service.ts b/client/platform/web-girder/api/rpc.service.ts index 82b3d48bf..2009cd150 100644 --- a/client/platform/web-girder/api/rpc.service.ts +++ b/client/platform/web-girder/api/rpc.service.ts @@ -37,8 +37,11 @@ function runTraining( path?: string; folderId?: string; }, + monitorEmail?: string, ) { - return girderRest.post('dive_rpc/train', { folderIds, labelText, fineTuneModel }, { + return girderRest.post('dive_rpc/train', { + folderIds, labelText, fineTuneModel, monitorEmail, + }, { params: { pipelineName, config, annotatedFramesOnly, }, diff --git a/client/platform/web-girder/views/RunTrainingMenu.vue b/client/platform/web-girder/views/RunTrainingMenu.vue index f270c41f3..b2e0be38a 100644 --- a/client/platform/web-girder/views/RunTrainingMenu.vue +++ b/client/platform/web-girder/views/RunTrainingMenu.vue @@ -7,7 +7,7 @@ import { useApi, TrainingConfigs } from 'dive-common/apispec'; import JobLaunchDialog from 'dive-common/components/JobLaunchDialog.vue'; import ImportButton from 'dive-common/components/ImportButton.vue'; import { useRequest } from 'dive-common/use'; -import { simplifyTrainingName } from 'dive-common/constants'; +import { simplifyTrainingName, isValidEmail } from 'dive-common/constants'; import { useBrand } from 'platform/web-girder/store/useBrand'; import { useConfig } from 'platform/web-girder/store/useConfig'; @@ -39,6 +39,13 @@ export default defineComponent({ const trainingConfigurations = ref(null); const selectedTrainingConfig = ref(null); const annotatedFramesOnly = ref(false); + const monitorEmail = ref(''); + const emailRules = [ + (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'), + ]; + const monitorEmailValid = computed(() => ( + !monitorEmail.value || isValidEmail(monitorEmail.value) + )); const fineTuning = ref(false); const selectedFineTune = ref(''); const { @@ -92,30 +99,22 @@ export default defineComponent({ async function runTrainingOnFolder() { const outputPipelineName = trainingOutputName.value; - if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value) { + if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value + || !monitorEmailValid.value) { return; } await _runTrainingRequest(() => { if (!trainingConfigurations.value || !selectedTrainingConfig.value) { throw new Error('Training configurations not found.'); } - if (labelText.value) { - return runTraining( - props.selectedDatasetIds, - outputPipelineName, - selectedTrainingConfig.value, - annotatedFramesOnly.value, - labelText.value, - selectedFineTuneObject.value, - ); - } return runTraining( props.selectedDatasetIds, outputPipelineName, selectedTrainingConfig.value, annotatedFramesOnly.value, - undefined, + labelText.value || undefined, selectedFineTuneObject.value, + monitorEmail.value.trim() || undefined, ); }); menuOpen.value = false; @@ -140,6 +139,9 @@ export default defineComponent({ brandData, trainingConfigurations, selectedTrainingConfig, + monitorEmail, + emailRules, + monitorEmailValid, annotatedFramesOnly, trainingOutputName, menuOpen, @@ -316,12 +318,22 @@ export default defineComponent({ hint="Model to Fine Tune" persistent-hint /> + Train on {{ selectedDatasetIds.length }} dataset(s) diff --git a/docs/Pipeline-Documentation.md b/docs/Pipeline-Documentation.md index ff72841fe..ae4b68f0c 100644 --- a/docs/Pipeline-Documentation.md +++ b/docs/Pipeline-Documentation.md @@ -111,6 +111,12 @@ By default, all classes from all input datasets are preserved in the output mode By default, training runs include all frames from the chosen input datasets, and frames without annotations are considered negative examples. If you choose to use annotated frames only, frames or images with zero annotations will be discarded. This option is useful for trying to train on datasets that are only partially annotated. +#### Email progress reports to + +This **optional** address receives reports from VIAME's training monitor while the run is in progress: a test message when training starts, validation statistics every few epochs, a notice when a possible error or deadlock is detected, and a final message saying whether the run finished normally. A copy of the training output is kept as `train.log` in the run's output directory alongside the `monitor_status.log` trail. + +Mail delivery is configured on the machine that runs training, not in DIVE. Set `VIAME_SMTP_SERVER` (as `host:port`) and, when the server needs a login, `VIAME_SMTP_USER` and `VIAME_SMTP_PASSWORD` in the environment of DIVE Desktop or of the training worker. On Linux a local `sendmail` is used when no server is set. Without either, the reports are still written to the status trail in the output directory. This option requires a VIAME build that includes the `viame monitor` tool. + ### Configurations | Configuration | Availability | Use Case | diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 13597d3ce..ca7568447 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -51,6 +51,7 @@ class RunTrainingArgs(BaseModel): folderIds: List[str] labelText: Optional[str] fineTuneModel: Optional[types.TrainingModelTuneArgs] + monitorEmail: Optional[str] = None class ScoringSourceModel(BaseModel): @@ -664,6 +665,7 @@ def run_training( 'annotated_frames_only': annotatedFramesOnly, 'label_txt': bodyParams.labelText, 'model': fineTuneModel, + 'monitor_email': bodyParams.monitorEmail or None, 'user_id': user.get('_id', 'unknown'), 'user_login': user.get('login', 'unknown'), 'force_transcoded': force_transcoded, diff --git a/server/dive_tasks/run_training.py b/server/dive_tasks/run_training.py index 57e7f11ec..ad57f18b8 100644 --- a/server/dive_tasks/run_training.py +++ b/server/dive_tasks/run_training.py @@ -156,6 +156,13 @@ def train_pipeline(self: Task, params: TrainingJob): if annotated_frames_only: command.append("--gt-frames-only") + # VIAME starts its training monitor next to the run, which emails + # progress and completion reports. Mail delivery is configured on the + # worker through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / _PASSWORD). + if params.get('monitor_email'): + command.append("--monitor-email") + command.append(shlex.quote(str(params['monitor_email']))) + if label_text: labels_path = input_path / "labels.txt" with open(labels_path, "w+") as labels_file: diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index a7d066ce4..e1bb18d91 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -205,6 +205,7 @@ class TrainingJob(TypedDict): annotated_frames_only: bool # Train on only the annotated frames label_txt: Optional[str] # Contents of a labels.txt to include in training model: Optional[TrainingModelTuneArgs] # Model for fine-tune training + monitor_email: Optional[str] # Address for progress reports from viame monitor user_id: str # user id who started the job user_login: str # login of user who started the kjob force_transcoded: Optional[bool] # Force using the transcoded version From 7b397142723ad9ff06c547363a1d17aca153aa46 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Fri, 11 Sep 2026 23:36:15 -0400 Subject: [PATCH 2/2] Align training email and label file options --- .../frontend/components/MultiTrainingMenu.vue | 39 +++++++-------- .../web-girder/views/RunTrainingMenu.vue | 48 +++++++++++-------- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index ae4aa7370..4e8f12ef4 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -474,13 +474,13 @@ export default defineComponent({ class="my-4 pt-0" dense > - + - + + +
- - - - -
diff --git a/client/platform/web-girder/views/RunTrainingMenu.vue b/client/platform/web-girder/views/RunTrainingMenu.vue index b2e0be38a..3897f5541 100644 --- a/client/platform/web-girder/views/RunTrainingMenu.vue +++ b/client/platform/web-girder/views/RunTrainingMenu.vue @@ -283,15 +283,34 @@ export default defineComponent({ {{ simplifyTrainingName(item.name || item) }} - + + + + + + + + - +