From 57b583e3d9a75b3b054d968f5f7a1c23339cd343 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Tue, 11 Aug 2026 11:07:50 +0300 Subject: [PATCH 1/2] Gaussian trsh: name the galloc failure and the memory cap correctly A `galloc: could not allocate memory.` line only appears in a Gaussian output file, and a Gaussian output file only exists if the job started, which means the scheduler granted the reservation it asked for. galloc therefore always means Gaussian was given too little memory, never too much: it asked the OS for what %mem promised and did not get it. ARC's observable over-allocation signal is a different one - the scheduler job log ("using less than N percent of requested"), classified by determine_job_log_memory_issues as "Memory requested is too high" and already excluded from this branch by the `'too high' not in error` guard. determine_ess_status called it 'Memory allocation failed (did you ask for too much?)'. That question has the sign backwards, and it is what led a later change to invert the remedy and reduce the request. The remedy itself - double the total memory, bounded by servers[server]['memory'] * job_max_server_node_memory_ allocation - is right and is unchanged. What was wrong at the cap was only the advice. "Use a higher-memory node" blames a node that was never the constraint: servers[server]['memory'] is a deliberate policy cap on how large one job may grow, so reaching it is the policy working, and moving the job to a physically larger node changes nothing because ARC would still request the same amount. Report that the job needs more than the configured cap and name the three levers that actually move it: servers[]['memory'], job_max_server_node_memory_allocation, or a cheaper method or level of theory. Nothing here trades cpu cores for memory per core. Gaussian's %mem is one total pool for the job: GaussianAdapter.set_input_file_memory derives it from submit_script_memory_mib alone, with no cpu_cores term, and cpu_cores reaches only %NProcShared. Halving the cores leaves %mem identical and resubmits the same failing job. Orca is the opposite case - it writes %maxcore, which is per-core, which is why its own trsh branch may legitimately trade cores. Escalation from 42 GB on 12 cores of a 240 GB server (cap 228.00 GB): 84 GB, 168 GB, 228 GB, then give up naming the cap and the three levers, with the cpu core count untouched throughout. The galloc.out fixture is a real Gaussian log of the failure, taken from arcbench's 3945043e; cherry-picking this commit onto arcbench will conflict on that file. --- arc/job/trsh.py | 16 ++-- arc/job/trsh_test.py | 47 +++++++++++- arc/testing/trsh/gaussian/galloc.out | 109 +++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 arc/testing/trsh/gaussian/galloc.out diff --git a/arc/job/trsh.py b/arc/job/trsh.py index 4a970dc840..e3ddcbce45 100644 --- a/arc/job/trsh.py +++ b/arc/job/trsh.py @@ -202,7 +202,7 @@ def determine_ess_status(output_path: str, line = '' elif 'malloc failed' in line or 'galloc' in line: keywords = ['Memory'] - error = 'Memory allocation failed (did you ask for too much?)' + error = 'Gaussian could not allocate the memory it was given, it requires more memory.' line = '' elif 'PGFIO/stdio: No such file or directory' in line: keywords = ['Scratch'] @@ -989,14 +989,16 @@ def trsh_ess_job(label: str, else: couldnt_trsh = True output_errors.append( - f'Error: Could not troubleshoot {job_type} for {label}! Gaussian exhausted memory even after ARC ' - f'reached the configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) while ' - f'still reserving scheduler headroom. Use a higher-memory node or lower the job cost; ' + f'Error: Could not troubleshoot {job_type} for {label}! Gaussian could not allocate the memory it ' + f'was given, and this job already requests the configured per-job memory cap on {server}, ' + f'{max_mem_allocation:.2f} GB. It needs more memory than the cap allows. Either raise the cap by ' + f'changing servers[\'{server}\'][\'memory\'] (currently {max_mem} GB) or ' + f'job_max_server_node_memory_allocation, or use a cheaper method or level of theory; ' ) logger.error( - f'Could not troubleshoot {job_type} job in {software} for {label}. ARC already reached the ' - f'configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) and still preserved ' - f'Gaussian headroom.' + f'Could not troubleshoot {job_type} job in {software} for {label}. Gaussian could not allocate ' + f'{memory_gb} GB, which is the configured per-job memory cap on {server} ' + f'({max_mem_allocation:.2f} GB).' ) if attempted_ess_trsh_methods: diff --git a/arc/job/trsh_test.py b/arc/job/trsh_test.py index 80fdf935e4..f9a196f2b7 100644 --- a/arc/job/trsh_test.py +++ b/arc/job/trsh_test.py @@ -433,7 +433,7 @@ def test_trsh_ess_job(self): # Gaussian: test 6 job_status = {'keywords': ['Memory', 'max_total_job_memory'], - 'error': 'Memory allocation failed (did you ask for too much?)'} + 'error': 'Gaussian could not allocate the memory it was given, it requires more memory.'} capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \ settings['servers']['server2']['memory'] output_errors, ess_trsh_methods, remove_checkfile, level_of_theory, software, job_type, fine, trsh_keyword, \ @@ -443,7 +443,8 @@ def test_trsh_ess_job(self): self.assertTrue(couldnt_trsh) self.assertEqual(memory, capped_memory_gb) - self.assertIn('Use a higher-memory node or lower the job cost', output_errors[0]) + self.assertIn('job_max_server_node_memory_allocation', output_errors[0]) + self.assertNotIn('higher-memory node', output_errors[0]) # Gaussian: test 7 - part 1 job_status = {'keywords': ['SCF', 'GL502', 'NoSymm']} @@ -832,6 +833,48 @@ def test_trsh_ess_job(self): num_heavy_atoms, cpu_cores, ess_trsh_methods, is_h=True, is_monoatomic=True) + def test_trsh_ess_job_gaussian_galloc_escalates_memory_to_the_cap(self): + """Test that a Gaussian galloc failure escalates the job memory up to the configured cap, then gives up.""" + label, level_of_theory, job_type, software = 'TS0', {'method': 'wb97xd', 'basis': 'def2tzvp'}, 'conf_opt', \ + 'gaussian' + server, num_heavy_atoms, fine = 'server2', 5, False + path = os.path.join(self.base_path['gaussian'], 'galloc.out') + status, keywords, error, _ = trsh.determine_ess_status(output_path=path, + species_label=label, + job_type=job_type, + software=software) + self.assertEqual(status, 'errored') + self.assertEqual(keywords, ['Memory']) + self.assertNotIn('too high', error) + self.assertNotIn('too much', error) + job_status = {'keywords': keywords, 'error': error} + capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \ + settings['servers']['server2']['memory'] + + memory, cpu_cores, ess_trsh_methods = 42.0, 12, [] + for expected_memory in [84.0, 168.0, capped_memory_gb]: + output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \ + trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory, + num_heavy_atoms, cpu_cores, ess_trsh_methods) + self.assertFalse(couldnt_trsh) + self.assertEqual(memory, expected_memory) + self.assertEqual(cpu_cores, 12) + self.assertEqual(ess_trsh_methods[-1], 'memory') + self.assertEqual(output_errors, []) + + output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \ + trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory, + num_heavy_atoms, cpu_cores, ess_trsh_methods) + self.assertTrue(couldnt_trsh) + self.assertEqual(memory, capped_memory_gb) + self.assertEqual(cpu_cores, 12) + self.assertIn('all_attempted', ess_trsh_methods) + self.assertIn(f'{capped_memory_gb:.2f} GB', output_errors[0]) + self.assertIn("servers['server2']['memory']", output_errors[0]) + self.assertIn('job_max_server_node_memory_allocation', output_errors[0]) + self.assertIn('cheaper method or level of theory', output_errors[0]) + self.assertNotIn('higher-memory node', output_errors[0]) + def test_trsh_ess_job_terachem_trsh_attempt_only(self): """Isolate the terachem trsh_attempt-only case from Gaussian stateful flow.""" label = 'ethanol' diff --git a/arc/testing/trsh/gaussian/galloc.out b/arc/testing/trsh/gaussian/galloc.out new file mode 100644 index 0000000000..c1b13aab6f --- /dev/null +++ b/arc/testing/trsh/gaussian/galloc.out @@ -0,0 +1,109 @@ + Entering Gaussian System, Link 0=g16 + Initial command: + /usr/local/g16/l1.exe "/gtmp/alon/scratch/g16/4393461.zeus-master/Gau-173582.inp" -scrdir="/gtmp/alon/scratch/g16/4393461.zeus-master/" + Entering Link 1 = /usr/local/g16/l1.exe PID= 173583. + + Copyright (c) 1988-2019, Gaussian, Inc. All Rights Reserved. + + This is part of the Gaussian(R) 16 program. It is based on + the Gaussian(R) 09 system (copyright 2009, Gaussian, Inc.), + the Gaussian(R) 03 system (copyright 2003, Gaussian, Inc.), + the Gaussian(R) 98 system (copyright 1998, Gaussian, Inc.), + the Gaussian(R) 94 system (copyright 1995, Gaussian, Inc.), + the Gaussian 92(TM) system (copyright 1992, Gaussian, Inc.), + the Gaussian 90(TM) system (copyright 1990, Gaussian, Inc.), + the Gaussian 88(TM) system (copyright 1988, Gaussian, Inc.), + the Gaussian 86(TM) system (copyright 1986, Carnegie Mellon + University), and the Gaussian 82(TM) system (copyright 1983, + Carnegie Mellon University). Gaussian is a federally registered + trademark of Gaussian, Inc. + + This software contains proprietary and confidential information, + including trade secrets, belonging to Gaussian, Inc. + + This software is provided under written license and may be + used, copied, transmitted, or stored only in accord with that + written license. + + The following legend is applicable only to US Government + contracts under FAR: + + RESTRICTED RIGHTS LEGEND + + Use, reproduction and disclosure by the US Government is + subject to restrictions as set forth in subparagraphs (a) + and (c) of the Commercial Computer Software - Restricted + Rights clause in FAR 52.227-19. + + Gaussian, Inc. + 340 Quinnipiac St., Bldg. 40, Wallingford CT 06492 + + + --------------------------------------------------------------- + Warning -- This program may not be used in any manner that + competes with the business of Gaussian, Inc. or will provide + assistance to any competitor of Gaussian, Inc. The licensee + of this program is prohibited from giving any competitor of + Gaussian, Inc. access to this program. By using this program, + the user acknowledges that Gaussian, Inc. is engaged in the + business of creating and licensing software in the field of + computational chemistry and represents and warrants to the + licensee that it is not a competitor of Gaussian, Inc. and that + it will not use this program in any manner prohibited above. + --------------------------------------------------------------- + + + Cite this work as: + Gaussian 16, Revision C.01, + M. J. Frisch, G. W. Trucks, H. B. Schlegel, G. E. Scuseria, + M. A. Robb, J. R. Cheeseman, G. Scalmani, V. Barone, + G. A. Petersson, H. Nakatsuji, X. Li, M. Caricato, A. V. Marenich, + J. Bloino, B. G. Janesko, R. Gomperts, B. Mennucci, H. P. Hratchian, + J. V. Ortiz, A. F. Izmaylov, J. L. Sonnenberg, D. Williams-Young, + F. Ding, F. Lipparini, F. Egidi, J. Goings, B. Peng, A. Petrone, + T. Henderson, D. Ranasinghe, V. G. Zakrzewski, J. Gao, N. Rega, + G. Zheng, W. Liang, M. Hada, M. Ehara, K. Toyota, R. Fukuda, + J. Hasegawa, M. Ishida, T. Nakajima, Y. Honda, O. Kitao, H. Nakai, + T. Vreven, K. Throssell, J. A. Montgomery, Jr., J. E. Peralta, + F. Ogliaro, M. J. Bearpark, J. J. Heyd, E. N. Brothers, K. N. Kudin, + V. N. Staroverov, T. A. Keith, R. Kobayashi, J. Normand, + K. Raghavachari, A. P. Rendell, J. C. Burant, S. S. Iyengar, + J. Tomasi, M. Cossi, J. M. Millam, M. Klene, C. Adamo, R. Cammi, + J. W. Ochterski, R. L. Martin, K. Morokuma, O. Farkas, + J. B. Foresman, and D. J. Fox, Gaussian, Inc., Wallingford CT, 2019. + + ****************************************** + Gaussian 16: EM64L-G16RevC.01 3-Jul-2019 + 4-Jul-2026 + ****************************************** + %chk=check.chk + %mem=61637mb + %NProcShared=16 + Will use up to 16 processors via shared memory. + -------------------------------------------------------- + #P opt=(calcfc) guess=read wb97xd/def2tzvp IOp(2/9=2000) + -------------------------------------------------------- + 1/10=4,18=20,19=15,26=3,38=1/1,3; + 2/9=2000,12=2,17=6,18=5,40=1/2; + 3/5=44,7=101,11=2,25=1,30=1,71=2,74=-58,116=-2,140=1/1,2,3; + 4/5=1/1; + 5/5=2,38=6/2; + 8/6=4,10=90,11=11/1; + 11/6=1,8=1,9=11,15=111,16=1/1,2,10; + 10/6=1,13=1/2; + 6/7=2,8=2,9=2,10=2,28=1/1; + 7/10=1,25=1/1,2,3,16; + 1/10=4,18=20,19=15,26=3/3(2); + 2/9=2000/2; + 99//99; + 2/9=2000/2; + 3/5=44,7=101,11=2,25=1,30=1,71=1,74=-58/1,2,3; + 4/5=5,16=3,69=1/1; + 5/5=2,38=5/2; + 7//1,2,3,16; + 1/18=20,19=15,26=3/3(-5); + 2/9=2000/2; + 6/7=2,8=2,9=2,10=2,19=2,28=1/1; + 99/9=1/99; + Leave Link 1 at Sat Jul 4 12:46:50 2026, MaxMem= 8078884864 cpu: 0.7 elap: 0.1 +galloc: could not allocate memory. From 79b7a6d6345c31efed98ba4d6826171a6bc33d89 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Tue, 11 Aug 2026 11:08:02 +0300 Subject: [PATCH 2/2] Apply the capped memory overhead to a job that requests exactly the cap set_cpu_and_mem clamps an over-large request to max_mem * job_max_server_node_memory_allocation and then, because that value is exactly the threshold it was just compared against with a strict `>`, falls through to the uncapped branch on every subsequent call. The clamped job is therefore given DEFAULT_JOB_MEMORY_OVERHEAD (1.10) instead of CAPPED_JOB_MEMORY_OVERHEAD (1.05), so the submit script asks the scheduler for 0.95 * 1.10 = 104.5% of the memory the server is configured to have, and the 'max_total_job_memory' keyword is never appended - making that keyword unreachable for any job that arrived at the cap through the troubleshooting loop, which is the only way a job gets there. trsh_ess_job clamps Gaussian memory to exactly this value, so the boundary is hit by ordinary troubleshooting rather than by an unusual input. On the 256 GB server2 fixture the request drops from 273941 MiB (267.5 GiB, more than the node has) to 261489 MiB (255.4 GiB). Only the Orca branch of trsh_ess_job reads 'max_total_job_memory'; the Gaussian branch carries it without acting on it, so newly emitting it changes no troubleshooting decision. --- arc/job/adapter.py | 4 ++-- arc/job/adapter_test.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/arc/job/adapter.py b/arc/job/adapter.py index 9f47ea0c6a..80f9c02176 100644 --- a/arc/job/adapter.py +++ b/arc/job/adapter.py @@ -589,9 +589,9 @@ def set_cpu_and_mem(self): self.cpu_cores = self.cpu_cores or job_cpu_cores max_mem = servers[self.server].get('memory', None) if self.server is not None else 32.0 # Max memory per node in GB. job_max_server_node_memory_allocation = default_job_settings.get('job_max_server_node_memory_allocation', 0.95) - if max_mem is not None and self.job_memory_gb > max_mem * job_max_server_node_memory_allocation: + if max_mem is not None and self.job_memory_gb >= max_mem * job_max_server_node_memory_allocation: logger.warning(f'The memory for job {self.job_name} using {self.job_adapter} ({self.job_memory_gb} GB) ' - f'exceeds {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' + f'reaches {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' f'{self.server}. Setting it to {job_max_server_node_memory_allocation * max_mem:.2f} GB.') self.job_memory_gb = job_max_server_node_memory_allocation * max_mem total_submit_script_memory_mib = math.ceil(self.job_memory_gb * MEMORY_GB_TO_MIB * CAPPED_JOB_MEMORY_OVERHEAD) diff --git a/arc/job/adapter_test.py b/arc/job/adapter_test.py index 3e3536ea09..24577a9a5c 100644 --- a/arc/job/adapter_test.py +++ b/arc/job/adapter_test.py @@ -22,7 +22,8 @@ from arc.level import Level from arc.species import ARCSpecies -servers, submit_filenames = settings['servers'], settings['submit_filenames'] +default_job_settings, servers, submit_filenames = \ + settings['default_job_settings'], settings['servers'], settings['submit_filenames'] class TestEnumerationClasses(unittest.TestCase): @@ -250,6 +251,23 @@ def test_set_cpu_and_mem(self): self.assertEqual(self.job_4.submit_script_memory, expected_memory) self.job_4.server = 'local' + capped_memory_gb = servers['server2']['memory'] \ + * default_job_settings['job_max_server_node_memory_allocation'] + original_job_memory_gb = self.job_4.job_memory_gb + self.addCleanup(setattr, self.job_4, 'job_memory_gb', original_job_memory_gb) + self.addCleanup(setattr, self.job_4, 'server', 'local') + self.job_4.server = 'server2' + self.job_4.cpu_cores = None + self.job_4.job_memory_gb = capped_memory_gb + self.job_4.set_cpu_and_mem() + self.assertEqual(self.job_4.job_memory_gb, capped_memory_gb) + self.assertEqual(self.job_4.submit_script_memory_mib, math.ceil(capped_memory_gb * 1024 * 1.05)) + self.assertLess(self.job_4.submit_script_memory_mib, servers['server2']['memory'] * 1024) + self.assertIn('max_total_job_memory', self.job_4.job_status[1]['keywords']) + self.job_4.job_status[1]['keywords'].remove('max_total_job_memory') + self.job_4.job_memory_gb = original_job_memory_gb + self.job_4.server = 'local' + def test_set_file_paths(self): """Test setting up the job's paths""" self.assertEqual(self.job_1.local_path, os.path.join(self.job_1.project_directory, 'calcs', 'Species',