Let the training dialogs email progress reports through viame monitor

client/dive-common/apispec.ts

@@ -454,6 +454,8 @@ interface Api {
       path?: string;
       folderId?: string;
     },
+    /** Address that receives progress and completion reports from the run. */
+    monitorEmail?: string,
   ): Promise<unknown>;
 
   /**

client/dive-common/constants.ts

@@ -242,6 +242,11 @@ function simplifyTrainingName(item: string) {
   return item.replace('.conf', '');
 }
 
+/** Loose check for a single mailbox address, enough to catch typos in forms. */
+function isValidEmail(address: string) {
+  return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(address.trim());
+}
+
 export {
   DefaultVideoFPS,
   ImageSequenceType,
@@ -284,4 +289,5 @@ export {
   pipelineCreatesDatasetMarkers,
   JsonConfigRegEx,
   simplifyTrainingName,
+  isValidEmail,
 };

client/platform/desktop/backend/native/viame.ts

@@ -956,6 +956,14 @@ async function train(
     command.push(labelsPath);
   }
 
+  // VIAME starts its training monitor next to the run, which emails progress
+  // and completion reports. Mail delivery is configured on the machine
+  // through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / VIAME_SMTP_PASSWORD).
+  if (runTrainingArgs.monitorEmail) {
+    command.push('--monitor-email');
+    command.push(`"${runTrainingArgs.monitorEmail}"`);
+  }
+
   const job = observeChild(spawn(command.join(' '), {
     shell: viameConstants.shell,
     cwd: jobWorkDir,

client/platform/desktop/constants.ts

@@ -234,6 +234,8 @@ export interface RunTraining extends JobArgs {
   };
   // working directory of a prior interrupted run to continue from
   resumeWorkingDir?: string;
+  // address that receives progress and completion reports from the run
+  monitorEmail?: string;
 }
 
 export interface RunScoring extends JobArgs, ScoringJobArgs {

client/platform/desktop/frontend/api.ts

@@ -270,6 +270,7 @@ async function runTraining(
     path?: string;
     folderId?: string;
   },
+  monitorEmail?: string,
 ): Promise<void> {
   const args: RunTraining = {
     type: JobType.RunTraining,
@@ -279,6 +280,7 @@ async function runTraining(
     annotatedFramesOnly,
     labelText,
     fineTuneModel,
+    monitorEmail,
   };
   gpuJobQueue.enqueue(args);
 }

client/platform/desktop/frontend/components/MultiTrainingMenu.vue

@@ -15,7 +15,7 @@ import {
   DatasetConfig, Pipelines, TrainingConfigs, useApi, Pipe,
 } from 'dive-common/apispec';
 import { usePrompt } from 'dive-common/vue-utilities/prompt-service';
-import { itemsPerPageOptions, simplifyTrainingName } from 'dive-common/constants';
+import { itemsPerPageOptions, simplifyTrainingName, isValidEmail } from 'dive-common/constants';
 import { clientSettings } from 'dive-common/store/settings';
 
 import { useRoute, useRouter } from 'vue-router/composables';
@@ -126,6 +126,9 @@ export default defineComponent({
     const nameRules = [
       (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'),
     ];
+    const emailRules = [
+      (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'),
+    ];
 
     const data = reactive({
       stagedItems: {} as Record<string, DatasetConfig>,
@@ -141,6 +144,7 @@ export default defineComponent({
         models: {},
       } as TrainingConfigs,
       annotatedFramesOnly: false,
+      monitorEmail: '',
     });
 
     const headersTmpl: DataTableHeader[] = [
@@ -210,6 +214,7 @@ export default defineComponent({
       stagedItems.value.length > 0
         && data.selectedTrainingConfig
         && data.trainingOutputName
+        && (!data.monitorEmail || isValidEmail(data.monitorEmail))
     ));
 
     async function deleteModel(item: Pipe) {
@@ -285,6 +290,7 @@ export default defineComponent({
           data.annotatedFramesOnly,
           labelText.value || undefined,
           foundTrainingModel,
+          data.monitorEmail.trim() || undefined,
         );
         router.push({ name: 'jobs' });
       } catch (err) {
@@ -352,6 +358,7 @@ export default defineComponent({
       resumeJob,
       discardJob,
       nameRules,
+      emailRules,
       itemsPerPageOptions,
       clientSettings,
       modelNames,
@@ -467,21 +474,33 @@ export default defineComponent({
         class="my-4 pt-0"
         dense
       >
-        <v-col sm="5">
+        <v-col cols="12" sm="6">
           <v-file-input
             v-model="labelFile"
             icon="mdi-folder-open"
-            label="Labels.txt mapping file (optional)"
-            hint="Combine or rename output classes using a labels.txt file"
-            persistant-hint
+            label="Labels .txt, .csv, or .json (optional)"
+            hint="Combine or rename output classes using a labels .txt, .csv, or .json file"
+            persistent-hint
             dense
             outlined
             hide-details
             clearable
             @click:clear="clearLabelText"
           />
         </v-col>
-        <v-spacer />
+        <v-col cols="12" sm="6">
+          <v-text-field
+            v-model="data.monitorEmail"
+            :rules="emailRules"
+            prepend-icon="mdi-email-outline"
+            outlined
+            dense
+            clearable
+            label="Email progress reports to (optional)"
+            hint="Sends training progress, error and completion reports; requires mail to be configured for VIAME"
+            persistent-hint
+          />
+        </v-col>
       </v-row>
       <v-data-table
         v-bind="{ headers: staged.headers, items: staged.items.value }"

client/platform/web-girder/api/rpc.service.ts

@@ -37,8 +37,11 @@ function runTraining(
     path?: string;
     folderId?: string;
   },
+  monitorEmail?: string,
 ) {
-  return girderRest.post('dive_rpc/train', { folderIds, labelText, fineTuneModel }, {
+  return girderRest.post('dive_rpc/train', {
+    folderIds, labelText, fineTuneModel, monitorEmail,
+  }, {
     params: {
       pipelineName, config, annotatedFramesOnly,
     },

client/platform/web-girder/views/RunTrainingMenu.vue

@@ -7,7 +7,7 @@ import { useApi, TrainingConfigs } from 'dive-common/apispec';
 import JobLaunchDialog from 'dive-common/components/JobLaunchDialog.vue';
 import ImportButton from 'dive-common/components/ImportButton.vue';
 import { useRequest } from 'dive-common/use';
-import { simplifyTrainingName } from 'dive-common/constants';
+import { simplifyTrainingName, isValidEmail } from 'dive-common/constants';
 import { useBrand } from 'platform/web-girder/store/useBrand';
 import { useConfig } from 'platform/web-girder/store/useConfig';
 
@@ -39,6 +39,13 @@ export default defineComponent({
     const trainingConfigurations = ref<TrainingConfigs | null>(null);
     const selectedTrainingConfig = ref<string | null>(null);
     const annotatedFramesOnly = ref<boolean>(false);
+    const monitorEmail = ref<string>('');
+    const emailRules = [
+      (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'),
+    ];
+    const monitorEmailValid = computed(() => (
+      !monitorEmail.value || isValidEmail(monitorEmail.value)
+    ));
     const fineTuning = ref<boolean>(false);
     const selectedFineTune = ref<string>('');
     const {
@@ -92,30 +99,22 @@ export default defineComponent({
 
     async function runTrainingOnFolder() {
       const outputPipelineName = trainingOutputName.value;
-      if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value) {
+      if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value
+        || !monitorEmailValid.value) {
         return;
       }
       await _runTrainingRequest(() => {
         if (!trainingConfigurations.value || !selectedTrainingConfig.value) {
           throw new Error('Training configurations not found.');
         }
-        if (labelText.value) {
-          return runTraining(
-            props.selectedDatasetIds,
-            outputPipelineName,
-            selectedTrainingConfig.value,
-            annotatedFramesOnly.value,
-            labelText.value,
-            selectedFineTuneObject.value,
-          );
-        }
         return runTraining(
           props.selectedDatasetIds,
           outputPipelineName,
           selectedTrainingConfig.value,
           annotatedFramesOnly.value,
-          undefined,
+          labelText.value || undefined,
           selectedFineTuneObject.value,
+          monitorEmail.value.trim() || undefined,
         );
       });
       menuOpen.value = false;
@@ -140,6 +139,9 @@ export default defineComponent({
       brandData,
       trainingConfigurations,
       selectedTrainingConfig,
+      monitorEmail,
+      emailRules,
+      monitorEmailValid,
       annotatedFramesOnly,
       trainingOutputName,
       menuOpen,
@@ -281,15 +283,34 @@ export default defineComponent({
                 {{ simplifyTrainingName(item.name || item) }}
               </template>
             </v-select>
-            <v-file-input
-              v-model="labelFile"
-              icon="mdi-folder-open"
-              label="Labels.txt mapping file (optional)"
-              hint="Combine or rename output classes using a labels.txt file"
-              persistent-hint
-              clearable
-              @click:clear="clearLabelText"
-            />
+            <v-row dense>
+              <v-col cols="12" sm="6">
+                <v-file-input
+                  v-model="labelFile"
+                  icon="mdi-folder-open"
+                  label="Labels .txt, .csv, or .json (optional)"
+                  hint="Combine or rename output classes using a labels .txt, .csv, or .json file"
+                  outlined
+                  dense
+                  persistent-hint
+                  clearable
+                  @click:clear="clearLabelText"
+                />
+              </v-col>
+              <v-col cols="12" sm="6">
+                <v-text-field
+                  v-model="monitorEmail"
+                  :rules="emailRules"
+                  prepend-icon="mdi-email-outline"
+                  outlined
+                  dense
+                  clearable
+                  label="Email progress reports to (optional)"
+                  hint="Sends training progress, error and completion reports; requires mail to be configured on the training worker"
+                  persistent-hint
+                />
+              </v-col>
+            </v-row>
             <v-checkbox
               v-model="annotatedFramesOnly"
               label="Use annotated frames only"
@@ -316,12 +337,13 @@ export default defineComponent({
               hint="Model to Fine Tune"
               persistent-hint
             />
+
             <v-btn
               depressed
               block
               color="primary"
               class="mt-4"
-              :disabled="!trainingOutputName || !selectedTrainingConfig"
+              :disabled="!trainingOutputName || !selectedTrainingConfig || !monitorEmailValid"
               @click="runTrainingOnFolder"
             >
               Train on {{ selectedDatasetIds.length }} dataset(s)

docs/Pipeline-Documentation.md

@@ -111,6 +111,12 @@ By default, all classes from all input datasets are preserved in the output mode
 
 By default, training runs include all frames from the chosen input datasets, and frames without annotations are considered negative examples.  If you choose to use annotated frames only, frames or images with zero annotations will be discarded.  This option is useful for trying to train on datasets that are only partially annotated.
 
+#### Email progress reports to
+
+This **optional** address receives reports from VIAME's training monitor while the run is in progress: a test message when training starts, validation statistics every few epochs, a notice when a possible error or deadlock is detected, and a final message saying whether the run finished normally.  A copy of the training output is kept as `train.log` in the run's output directory alongside the `monitor_status.log` trail.
+
+Mail delivery is configured on the machine that runs training, not in DIVE.  Set `VIAME_SMTP_SERVER` (as `host:port`) and, when the server needs a login, `VIAME_SMTP_USER` and `VIAME_SMTP_PASSWORD` in the environment of DIVE Desktop or of the training worker.  On Linux a local `sendmail` is used when no server is set.  Without either, the reports are still written to the status trail in the output directory.  This option requires a VIAME build that includes the `viame monitor` tool.
+
 ### Configurations
 
 | Configuration | Availability | Use Case |

server/dive_server/crud_rpc.py

@@ -51,6 +51,7 @@ class RunTrainingArgs(BaseModel):
     folderIds: List[str]
     labelText: Optional[str]
     fineTuneModel: Optional[types.TrainingModelTuneArgs]
+    monitorEmail: Optional[str] = None
 
 
 class ScoringSourceModel(BaseModel):
@@ -664,6 +665,7 @@ def run_training(
         'annotated_frames_only': annotatedFramesOnly,
         'label_txt': bodyParams.labelText,
         'model': fineTuneModel,
+        'monitor_email': bodyParams.monitorEmail or None,
         'user_id': user.get('_id', 'unknown'),
         'user_login': user.get('login', 'unknown'),
         'force_transcoded': force_transcoded,

server/dive_tasks/run_training.py

@@ -156,6 +156,13 @@ def train_pipeline(self: Task, params: TrainingJob):
         if annotated_frames_only:
             command.append("--gt-frames-only")
 
+        # VIAME starts its training monitor next to the run, which emails
+        # progress and completion reports. Mail delivery is configured on the
+        # worker through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / _PASSWORD).
+        if params.get('monitor_email'):
+            command.append("--monitor-email")
+            command.append(shlex.quote(str(params['monitor_email'])))
+
         if label_text:
             labels_path = input_path / "labels.txt"
             with open(labels_path, "w+") as labels_file:

server/dive_utils/types.py

@@ -205,6 +205,7 @@ class TrainingJob(TypedDict):
     annotated_frames_only: bool  # Train on only the annotated frames
     label_txt: Optional[str]  # Contents of a labels.txt to include in training
     model: Optional[TrainingModelTuneArgs]  # Model for fine-tune training
+    monitor_email: Optional[str]  # Address for progress reports from viame monitor
     user_id: str  # user id who started the job
     user_login: str  # login of user who started the kjob
     force_transcoded: Optional[bool]  # Force using the transcoded version