From 3379d6834c80833842e0d8ac1aa44b3c8f3ee0e0 Mon Sep 17 00:00:00 2001 From: Peter Willendrup Date: Fri, 23 Oct 2026 13:01:48 +0200 Subject: [PATCH 1/4] mctest: --scan_split=auto for %Scan tests only, not with --statfactor 6253f3fe2 (--statfactor) put its block between the %Scan branch and the --scan_split=auto that belongs to it, so the option was added to every test when --statfactor was given (a %Example then failed in mcrun with "No interval range specified"), and to no %Scan test without it. Co-Authored-By: Claude Opus 5.5 --- tools/Python/mctest/mctest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/Python/mctest/mctest.py b/tools/Python/mctest/mctest.py index 1162bd982..4b95d6d35 100644 --- a/tools/Python/mctest/mctest.py +++ b/tools/Python/mctest/mctest.py @@ -578,11 +578,11 @@ def mccode_test(branchdir, testdir, limitinstrs=None, instrfilter=None, compfilt if test.scan: testncount = test.ncount or "%g" % min(float(ncount), 1e5) timeout = runmax * len(test.targetval) + if mpi is None: + parvals = parvals + " --scan_split=auto" if statfactor != 1: testncount = "%g" % (float(testncount) * statfactor) timeout = timeout * max(1, statfactor) - if mpi is None: - parvals = parvals + " --scan_split=auto" # Did test run already? if not os.path.exists(join(testdir, test.instrname, str(test.testnb))): if nexus: From a44a668e5e9ea98df7391604bcfc22501f3fcd5c Mon Sep 17 00:00:00 2001 From: Peter Willendrup Date: Fri, 23 Oct 2026 13:17:24 +0200 Subject: [PATCH 2/4] mcrun --scan_split: merge NeXus steps as a serial NeXus scan writes them Each parallel step writes its own dir/i/mccode.h5 with its data as entry1. These were merged into dir/mccode.h5 with h5copy (/entry1 -> /entryN), which keeps references to entry1 and the step dir: - the 'target' attribute of each linked monitor in entry2.. pointed to /entry1/instrument/...: nexusformat/NeXpy follow it and showed entry1's data for every scan step (h5py-based readers were not affected) - Directory/name/title/file_name named the step dir dir/i - the step dirs with their mccode.h5 copies were left behind merge_nexus_steps() now copies the entries with h5py, rewrites those references (keeping fixed-length strings) and removes the step dirs, so the result matches a serial NeXus scan item by item apart from times. h5py is needed for NeXus --scan_split scans instead of the h5copy tool. Co-Authored-By: Claude Opus 5.5 --- tools/Python/mcrun/optimisation.py | 55 ++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/tools/Python/mcrun/optimisation.py b/tools/Python/mcrun/optimisation.py index 7ba7b8f82..2cd64ff58 100644 --- a/tools/Python/mcrun/optimisation.py +++ b/tools/Python/mcrun/optimisation.py @@ -633,6 +633,50 @@ def run(self): LOG.info('Scan complete: all %d point(s) succeeded.', len(points)) +def merge_nexus_steps(mcstas_dir, indices): + """ Merges the mccode.h5 of each --scan_split step dir mcstas_dir/i into + mcstas_dir/mccode.h5 as entry1..N, as a serial NeXus scan writes it, and + removes the step dirs. Each step wrote its data as its own entry1 in its + own dir, so the copied entries refer to those: the 'target' attribute of + each linked monitor (e.g. /entry1/instrument/... in entry2, which NeXus + readers such as nexusformat/NeXpy follow, showing entry1's data), and the + step dir in the simulation metadata. Both are rewritten here. """ + import os, shutil + import h5py + import numpy as np + # step dirs as passed to the step runs, see _simulate_point() + stepdirs = [f'{mcstas_dir}/{i}' for i in indices if os.path.exists(f'{mcstas_dir}/{i}/mccode.h5')] + with h5py.File(join(mcstas_dir, 'mccode.h5'), 'w') as merged: + for n, stepdir in enumerate(stepdirs, 1): + + def fix(v, entry=True): + s = v.decode() if isinstance(v, bytes) else v + if entry: + s = re.sub(r'/entry1\b', f'/entry{n}', s) + s = s.replace(stepdir, mcstas_dir) + return np.bytes_(s.encode()) if isinstance(v, bytes) else s # fixed-length, as written by the runtime + + def fix_all(name, obj): + for k, v in obj.attrs.items(): + if isinstance(v, (bytes, str)): + obj.attrs[k] = fix(v) + if isinstance(obj, h5py.Dataset) and obj.dtype.kind == 'S' and obj.size == 1: + # fixed-length, null-terminated string (e.g. title): rewritten only + # when it changes, by the step dir fix, which shortens it so it fits + v = obj[()] if not obj.shape else obj[0] + if fix(v, False) != v: + obj[...] = fix(v, False) + + with h5py.File(f'{stepdir}/mccode.h5', 'r') as f: + if n == 1: + merged.attrs.update({k: fix(v) if isinstance(v, (bytes, str)) else v for k, v in f.attrs.items()}) + f.copy('entry1', merged, name=f'entry{n}') + fix_all(f'entry{n}', merged[f'entry{n}']) + merged[f'entry{n}'].visititems(fix_all) + for stepdir in stepdirs: + shutil.rmtree(stepdir) + + class Scanner_split: """ Perform a series of simulation steps along a given set of points, Where each simulation is controlled by its own thread. """ @@ -675,16 +719,7 @@ def run(self): if self.mcstas.options.format.lower() == 'nexus': # Parallel steps can't safely append to one HDF5 file, so each wrote # its own dir/i/mccode.h5 - merge them as entry1..N like a serial scan. - import os, shutil, subprocess - merged = join(mcstas_dir, 'mccode.h5') - steps = [join(mcstas_dir, str(r['index']), 'mccode.h5') for r in results] - steps = [s for s in steps if os.path.exists(s)] - for n, step in enumerate(steps): - if n == 0: - shutil.copy(step, merged) - else: - subprocess.run(['h5copy', '-i', step, '-o', merged, - '-s', '/entry1', '-d', f'/entry{n+1}'], check=True) + merge_nexus_steps(mcstas_dir, [r['index'] for r in results]) skipped = [r['index'] for r in results if not r['detectors']] From adb0259f9362155c46a67c715602ad72f75bbf6a Mon Sep 17 00:00:00 2001 From: Peter Willendrup Date: Fri, 23 Oct 2026 13:23:06 +0200 Subject: [PATCH 3/4] mcrun --scan_split: NeXus merge of successful steps only, keep dirs on failure A failed step can leave a partial, unreadable dir/i/mccode.h5: the merge (by file existence, also with h5copy before) then crashed mcrun before mccode.dat was written. Merge only the steps with detector values (the rows of mccode.dat), numbered entry1..N as a serial NeXus scan does, and keep all step dirs when a step failed, for inspection. Co-Authored-By: Claude Opus 5.5 --- tools/Python/mcrun/optimisation.py | 31 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tools/Python/mcrun/optimisation.py b/tools/Python/mcrun/optimisation.py index 2cd64ff58..02244acaf 100644 --- a/tools/Python/mcrun/optimisation.py +++ b/tools/Python/mcrun/optimisation.py @@ -633,19 +633,21 @@ def run(self): LOG.info('Scan complete: all %d point(s) succeeded.', len(points)) -def merge_nexus_steps(mcstas_dir, indices): - """ Merges the mccode.h5 of each --scan_split step dir mcstas_dir/i into - mcstas_dir/mccode.h5 as entry1..N, as a serial NeXus scan writes it, and - removes the step dirs. Each step wrote its data as its own entry1 in its - own dir, so the copied entries refer to those: the 'target' attribute of - each linked monitor (e.g. /entry1/instrument/... in entry2, which NeXus - readers such as nexusformat/NeXpy follow, showing entry1's data), and the - step dir in the simulation metadata. Both are rewritten here. """ - import os, shutil +def merge_nexus_steps(mcstas_dir, indices, keep_steps=False): + """ Merges the mccode.h5 of each (successful) --scan_split step dir + mcstas_dir/i into mcstas_dir/mccode.h5 as entry1..N, as a serial NeXus + scan writes it, and removes the step dirs unless keep_steps (e.g. when + a step failed, to leave its output for inspection). Each step wrote its + data as its own entry1 in its own dir, so the copied entries refer to + those: the 'target' attribute of each linked monitor (e.g. + /entry1/instrument/... in entry2, which NeXus readers such as + nexusformat/NeXpy follow, showing entry1's data), and the step dir in the + simulation metadata. Both are rewritten here. """ + import shutil import h5py import numpy as np # step dirs as passed to the step runs, see _simulate_point() - stepdirs = [f'{mcstas_dir}/{i}' for i in indices if os.path.exists(f'{mcstas_dir}/{i}/mccode.h5')] + stepdirs = [f'{mcstas_dir}/{i}' for i in indices] with h5py.File(join(mcstas_dir, 'mccode.h5'), 'w') as merged: for n, stepdir in enumerate(stepdirs, 1): @@ -673,6 +675,10 @@ def fix_all(name, obj): f.copy('entry1', merged, name=f'entry{n}') fix_all(f'entry{n}', merged[f'entry{n}']) merged[f'entry{n}'].visititems(fix_all) + if keep_steps: + LOG.info('NeXus scan steps merged into %s; step dirs kept since not all steps succeeded', + join(mcstas_dir, 'mccode.h5')) + return for stepdir in stepdirs: shutil.rmtree(stepdir) @@ -719,7 +725,10 @@ def run(self): if self.mcstas.options.format.lower() == 'nexus': # Parallel steps can't safely append to one HDF5 file, so each wrote # its own dir/i/mccode.h5 - merge them as entry1..N like a serial scan. - merge_nexus_steps(mcstas_dir, [r['index'] for r in results]) + # A failed step can leave a partial, unreadable mccode.h5: merge only the + # steps that gave detector values (the rows of mccode.dat) + ok = [r['index'] for r in results if r['detectors']] + merge_nexus_steps(mcstas_dir, ok, keep_steps=len(ok) < len(results)) skipped = [r['index'] for r in results if not r['detectors']] From 898b906ae4a3a4ad74be667739ef1b811b8f30c8 Mon Sep 17 00:00:00 2001 From: Peter Willendrup Date: Fri, 23 Oct 2026 13:47:46 +0200 Subject: [PATCH 4/4] mcrun --scan_split: mcrun logging and i/N step numbers in the workers The step workers of a split scan run in their own processes. With the spawn start method (macOS, Windows) they lacked mcrun's log setup, so their warnings were printed bare (no "WARNING:") and their INFO messages were lost; they also numbered steps from 0 without the total. A Pool initializer now sets up mcrun's logging where the worker has none (fork, Linux, already inherits it), and the step messages read "Scan step i+1/N", as in the serial scan. Co-Authored-By: Claude Opus 5.5 --- tools/Python/mcrun/optimisation.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tools/Python/mcrun/optimisation.py b/tools/Python/mcrun/optimisation.py index 02244acaf..72c9f2639 100644 --- a/tools/Python/mcrun/optimisation.py +++ b/tools/Python/mcrun/optimisation.py @@ -1,5 +1,5 @@ from os.path import basename -from log import McRunException, getLogger +from log import McRunException, getLogger, setupLogger from datetime import datetime from decimal import Decimal from os.path import join @@ -410,8 +410,16 @@ def scan_base_seed(mcstas, intervals): LOG.info('Scan base seed: %d' % mcstas.options.seed) return mcstas.options.seed +def _init_worker(): + """ mcrun's logging in a --scan_split worker process: needed with the spawn + start method (macOS, Windows), inherited already with fork (Linux) """ + import logging + if not logging.getLogger().handlers: + setupLogger() + + def _simulate_point(args): - i, point, intervals, mcstas_config, mcstas_dir, base_seed = args + i, npoints, point, intervals, mcstas_config, mcstas_dir, base_seed = args from shutil import copyfile from os.path import join @@ -455,8 +463,8 @@ def _simulate_point(args): # whose stdout didn't contain any "Detector: ..." lines at # all) - treated the same as a runtime failure below: skip # this point rather than writing an empty/malformed row. - LOG.warning('Scan step %d produced no detector data - skipping this point. Parameters were: %s', - i, ', '.join(f'{k}={v}' for k, v in point.items())) + LOG.warning('Scan step %d/%d produced no detector data - skipping this point. Parameters were: %s', + i + 1, npoints, ', '.join(f'{k}={v}' for k, v in point.items())) detectors = None except Exception as e: # A single failed scan point (simulation crash, non-zero exit, @@ -465,8 +473,8 @@ def _simulate_point(args): # already skips any result with detectors=None rather than writing # a row for it, so the scan carries on to the remaining points and # mccode.dat simply omits this one. - LOG.warning('Scan step %d failed (%s: %s) - skipping this point and continuing with the rest of the scan. ' - 'Parameters were: %s', i, type(e).__name__, e, ', '.join(f'{k}={v}' for k, v in point.items())) + LOG.warning('Scan step %d/%d failed (%s: %s) - skipping this point and continuing with the rest of the scan. ' + 'Parameters were: %s', i + 1, npoints, type(e).__name__, e, ', '.join(f'{k}={v}' for k, v in point.items())) detectors = None result = { @@ -711,12 +719,13 @@ def run(self): base_seed = scan_base_seed(self.mcstas, self.intervals) # Prepare data to pass into processes + points = list(self.points) args_list = [ - (i, point, self.intervals, self.mcstas, mcstas_dir, base_seed) - for i, point in enumerate(self.points) + (i, len(points), point, self.intervals, self.mcstas, mcstas_dir, base_seed) + for i, point in enumerate(points) ] - with Pool(processes=self.nb_cpu) as pool: + with Pool(processes=self.nb_cpu, initializer=_init_worker) as pool: results = pool.map(_simulate_point, args_list) # Sort results to preserve order