diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d4eed0dcd..e9af49d40 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -454,6 +454,8 @@ interface Api { path?: string; folderId?: string; }, + /** Address that receives progress and completion reports from the run. */ + monitorEmail?: string, ): Promise; /** diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index d949b067a..3545700e2 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -242,6 +242,11 @@ function simplifyTrainingName(item: string) { return item.replace('.conf', ''); } +/** Loose check for a single mailbox address, enough to catch typos in forms. */ +function isValidEmail(address: string) { + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(address.trim()); +} + export { DefaultVideoFPS, ImageSequenceType, @@ -284,4 +289,5 @@ export { pipelineCreatesDatasetMarkers, JsonConfigRegEx, simplifyTrainingName, + isValidEmail, }; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index a475c68d3..206c8bd04 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -956,6 +956,14 @@ async function train( command.push(labelsPath); } + // VIAME starts its training monitor next to the run, which emails progress + // and completion reports. Mail delivery is configured on the machine + // through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / VIAME_SMTP_PASSWORD). + if (runTrainingArgs.monitorEmail) { + command.push('--monitor-email'); + command.push(`"${runTrainingArgs.monitorEmail}"`); + } + const job = observeChild(spawn(command.join(' '), { shell: viameConstants.shell, cwd: jobWorkDir, diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 53caafd73..5ae8d1f69 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -234,6 +234,8 @@ export interface RunTraining extends JobArgs { }; // working directory of a prior interrupted run to continue from resumeWorkingDir?: string; + // address that receives progress and completion reports from the run + monitorEmail?: string; } export interface RunScoring extends JobArgs, ScoringJobArgs { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 825633d6d..124bf93d9 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -270,6 +270,7 @@ async function runTraining( path?: string; folderId?: string; }, + monitorEmail?: string, ): Promise { const args: RunTraining = { type: JobType.RunTraining, @@ -279,6 +280,7 @@ async function runTraining( annotatedFramesOnly, labelText, fineTuneModel, + monitorEmail, }; gpuJobQueue.enqueue(args); } diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index cf38b2cac..4e8f12ef4 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -15,7 +15,7 @@ import { DatasetConfig, Pipelines, TrainingConfigs, useApi, Pipe, } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; -import { itemsPerPageOptions, simplifyTrainingName } from 'dive-common/constants'; +import { itemsPerPageOptions, simplifyTrainingName, isValidEmail } from 'dive-common/constants'; import { clientSettings } from 'dive-common/store/settings'; import { useRoute, useRouter } from 'vue-router/composables'; @@ -126,6 +126,9 @@ export default defineComponent({ const nameRules = [ (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'), ]; + const emailRules = [ + (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'), + ]; const data = reactive({ stagedItems: {} as Record, @@ -141,6 +144,7 @@ export default defineComponent({ models: {}, } as TrainingConfigs, annotatedFramesOnly: false, + monitorEmail: '', }); const headersTmpl: DataTableHeader[] = [ @@ -210,6 +214,7 @@ export default defineComponent({ stagedItems.value.length > 0 && data.selectedTrainingConfig && data.trainingOutputName + && (!data.monitorEmail || isValidEmail(data.monitorEmail)) )); async function deleteModel(item: Pipe) { @@ -285,6 +290,7 @@ export default defineComponent({ data.annotatedFramesOnly, labelText.value || undefined, foundTrainingModel, + data.monitorEmail.trim() || undefined, ); router.push({ name: 'jobs' }); } catch (err) { @@ -352,6 +358,7 @@ export default defineComponent({ resumeJob, discardJob, nameRules, + emailRules, itemsPerPageOptions, clientSettings, modelNames, @@ -467,13 +474,13 @@ export default defineComponent({ class="my-4 pt-0" dense > - + - + + + (null); const selectedTrainingConfig = ref(null); const annotatedFramesOnly = ref(false); + const monitorEmail = ref(''); + const emailRules = [ + (val: string | null) => (!val || isValidEmail(val) || 'Enter a valid email address'), + ]; + const monitorEmailValid = computed(() => ( + !monitorEmail.value || isValidEmail(monitorEmail.value) + )); const fineTuning = ref(false); const selectedFineTune = ref(''); const { @@ -92,30 +99,22 @@ export default defineComponent({ async function runTrainingOnFolder() { const outputPipelineName = trainingOutputName.value; - if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value) { + if (trainingDisabled.value || !outputPipelineName || jobsDisabled.value + || !monitorEmailValid.value) { return; } await _runTrainingRequest(() => { if (!trainingConfigurations.value || !selectedTrainingConfig.value) { throw new Error('Training configurations not found.'); } - if (labelText.value) { - return runTraining( - props.selectedDatasetIds, - outputPipelineName, - selectedTrainingConfig.value, - annotatedFramesOnly.value, - labelText.value, - selectedFineTuneObject.value, - ); - } return runTraining( props.selectedDatasetIds, outputPipelineName, selectedTrainingConfig.value, annotatedFramesOnly.value, - undefined, + labelText.value || undefined, selectedFineTuneObject.value, + monitorEmail.value.trim() || undefined, ); }); menuOpen.value = false; @@ -140,6 +139,9 @@ export default defineComponent({ brandData, trainingConfigurations, selectedTrainingConfig, + monitorEmail, + emailRules, + monitorEmailValid, annotatedFramesOnly, trainingOutputName, menuOpen, @@ -281,15 +283,34 @@ export default defineComponent({ {{ simplifyTrainingName(item.name || item) }} - + + + + + + + + + Train on {{ selectedDatasetIds.length }} dataset(s) diff --git a/docs/Pipeline-Documentation.md b/docs/Pipeline-Documentation.md index ff72841fe..ae4b68f0c 100644 --- a/docs/Pipeline-Documentation.md +++ b/docs/Pipeline-Documentation.md @@ -111,6 +111,12 @@ By default, all classes from all input datasets are preserved in the output mode By default, training runs include all frames from the chosen input datasets, and frames without annotations are considered negative examples. If you choose to use annotated frames only, frames or images with zero annotations will be discarded. This option is useful for trying to train on datasets that are only partially annotated. +#### Email progress reports to + +This **optional** address receives reports from VIAME's training monitor while the run is in progress: a test message when training starts, validation statistics every few epochs, a notice when a possible error or deadlock is detected, and a final message saying whether the run finished normally. A copy of the training output is kept as `train.log` in the run's output directory alongside the `monitor_status.log` trail. + +Mail delivery is configured on the machine that runs training, not in DIVE. Set `VIAME_SMTP_SERVER` (as `host:port`) and, when the server needs a login, `VIAME_SMTP_USER` and `VIAME_SMTP_PASSWORD` in the environment of DIVE Desktop or of the training worker. On Linux a local `sendmail` is used when no server is set. Without either, the reports are still written to the status trail in the output directory. This option requires a VIAME build that includes the `viame monitor` tool. + ### Configurations | Configuration | Availability | Use Case | diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 13597d3ce..ca7568447 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -51,6 +51,7 @@ class RunTrainingArgs(BaseModel): folderIds: List[str] labelText: Optional[str] fineTuneModel: Optional[types.TrainingModelTuneArgs] + monitorEmail: Optional[str] = None class ScoringSourceModel(BaseModel): @@ -664,6 +665,7 @@ def run_training( 'annotated_frames_only': annotatedFramesOnly, 'label_txt': bodyParams.labelText, 'model': fineTuneModel, + 'monitor_email': bodyParams.monitorEmail or None, 'user_id': user.get('_id', 'unknown'), 'user_login': user.get('login', 'unknown'), 'force_transcoded': force_transcoded, diff --git a/server/dive_tasks/run_training.py b/server/dive_tasks/run_training.py index 57e7f11ec..ad57f18b8 100644 --- a/server/dive_tasks/run_training.py +++ b/server/dive_tasks/run_training.py @@ -156,6 +156,13 @@ def train_pipeline(self: Task, params: TrainingJob): if annotated_frames_only: command.append("--gt-frames-only") + # VIAME starts its training monitor next to the run, which emails + # progress and completion reports. Mail delivery is configured on the + # worker through VIAME_SMTP_SERVER (and VIAME_SMTP_USER / _PASSWORD). + if params.get('monitor_email'): + command.append("--monitor-email") + command.append(shlex.quote(str(params['monitor_email']))) + if label_text: labels_path = input_path / "labels.txt" with open(labels_path, "w+") as labels_file: diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index a7d066ce4..e1bb18d91 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -205,6 +205,7 @@ class TrainingJob(TypedDict): annotated_frames_only: bool # Train on only the annotated frames label_txt: Optional[str] # Contents of a labels.txt to include in training model: Optional[TrainingModelTuneArgs] # Model for fine-tune training + monitor_email: Optional[str] # Address for progress reports from viame monitor user_id: str # user id who started the job user_login: str # login of user who started the kjob force_transcoded: Optional[bool] # Force using the transcoded version