From 3a8252d0b179e034ade2bebd56aaf0072a895466 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Thu, 2 Apr 2026 14:59:23 +0100 Subject: [PATCH 1/9] add merger script --- fixedTarget/batch/gangaScript_merger.py | 19 ++++++ fixedTarget/batch/merger.py | 86 +++++++++++++++++++++++++ fixedTarget/batch/wn_script.py | 5 +- 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 fixedTarget/batch/gangaScript_merger.py create mode 100644 fixedTarget/batch/merger.py mode change 100644 => 100755 fixedTarget/batch/wn_script.py diff --git a/fixedTarget/batch/gangaScript_merger.py b/fixedTarget/batch/gangaScript_merger.py new file mode 100644 index 0000000..06466fe --- /dev/null +++ b/fixedTarget/batch/gangaScript_merger.py @@ -0,0 +1,19 @@ +jNo = 61 + +j = Job(name = f"Merge output of production job {jNo}") +j.virtualization = Apptainer(image="/cvmfs/unpacked.cern.ch/registry.cern.ch/ship/gha-runner:latest/") +j.virtualization.mounts = {'/cvmfs':'/cvmfs', '/home/hep/mesmith':'/home/hep/mesmith' } +j.application = Executable(exe = File('wn_script.py')) +j.inputfiles = [LocalFile("merger.py")] +myargs = ['merger.py' if _a=='run_fixedTarget.py' else _a for _a in jobs(jNo).application.args] +myargs.extend(['-j', jNo, '--prodSite', 'GRIDPP']) +myargs.append('-i') +inputfiles = jobs(jNo).backend.getOutputDataAccessURLs(protocol='"root"') +myargs.extend(inputfiles) +j.application.args = myargs +j.outputfiles = [LocalFile('pythia_*.root')] + +j.backend = Condor() +j.backend.cdf_options["+MaxRuntime"] = '3000' + +j.submit() diff --git a/fixedTarget/batch/merger.py b/fixedTarget/batch/merger.py new file mode 100644 index 0000000..d2aa37f --- /dev/null +++ b/fixedTarget/batch/merger.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +import os +import argparse +import ROOT +import uproot +import json +from datetime import datetime + +def mergeFiles(myfiles, tmpFile): + myChain = ROOT.TChain("cbmsim") + for _f in myfiles: + myChain.Add(_f) + myChain.Merge(tmpFile) + + +parser = argparse.ArgumentParser( + description="Run FairShip file merger" +) + +parser.add_argument("-i", "--inFiles", nargs="*", + help="input files to merge") + +parser.add_argument("-n", "--genEvents", + default=500000, + type=int, + help="How many events were generated to make this file" + ) + +parser.add_argument("-e", "--eCut", + default=30, + type=float, + help="What was the energy cut to produce the file" + ) + +parser.add_argument("-j", "--gangaJob", + type=int, + help="Which ganga job produced these files", + ) +parser.add_argument("-s", "--site", + default="CERN", + help="Which site was this run at" + ) + +parser.add_argument("-o", "--outputPath", + help="A placeholder") + +parser.add_argument("--prodSite", + default="CERN", + help="Which site was this run at" + ) + +args = parser.parse_args() + + +if args.prodSite == "GRIDPP": + os.environ["X509_USER_PROXY"]="/home/hep/mesmith/private/my_proxy.pem" + os.environ["X509_VOMSES"]="/cvmfs/grid.cern.ch/etc/grid-security/vomses" + os.environ["X509_VOMS_DIR"]="/cvmfs/grid.cern.ch/etc/grid-security/vomsdir" + os.environ["X509_CERT_DIR"]="/cvmfs/grid.cern.ch/etc/grid-security/certificates" + +files_to_merge = args.inFiles +if not len(files_to_merge)>0: + print("ERROR: no files to merge! Doing nothing") + sys.exit() +print("INFO: Merging %s files from job %s" % (len(files_to_merge), arg.gangaJob)) + +total_pot = len(files_to_merge) * args.genEvents +outName = f"pythia8_Geant4_eCut_{args.eCut}_PoT_{total_pot}_{args.site}_j{args.gangaJob}.root" + +mergeFiles(files_to_merge, outName) + +fsr = { + "PoT" : total_pot, + "EnergyCut": args.eCut, + "PoTperFile": args.genEvents, + "nFilesMerged": len(files_to_merge), + "filesMerged": args.inFiles, + "prodSite": args.prodSite, + "mergeDate": datetime.today().strftime('%Y-%m-%d') +} + +print("INFO: Adding the file summary") +with uproot.update(outName) as _f: + _f["FileSummary"] = json.dumps(fsr) + +print("INFO: All done") diff --git a/fixedTarget/batch/wn_script.py b/fixedTarget/batch/wn_script.py old mode 100644 new mode 100755 index 4309003..ab9899a --- a/fixedTarget/batch/wn_script.py +++ b/fixedTarget/batch/wn_script.py @@ -60,7 +60,8 @@ def main(): FS_INSTALL = args.fs_install or '/cvmfs/ship.cern.ch/' + args.cvmfs_version WORK_DIR = args.work_dir or f"{FS_INSTALL}/sw/" INIT_SCRIPT = args.init_script or f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/etc/profile.d/init.sh" - RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" +# RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" + RUN_SCRIPT=args.runfile fs_version = args.cvmfs_version fs_tag = args.FairShip_tag or args.cvmfs_version @@ -79,6 +80,8 @@ def main(): command = f""" export WORK_DIR="{WORK_DIR}" source "{INIT_SCRIPT}" + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd) + python "{RUN_SCRIPT}" {passthrough} """ From 98403053cbbe0fdd673c126763e2d9ba1e6f38b1 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Thu, 2 Apr 2026 15:46:36 +0100 Subject: [PATCH 2/9] add namespace function --- fixedTarget/batch/ganga.py | 55 ++++++++++++++++++++++++++++++++++ fixedTarget/batch/wn_script.py | 14 +++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 fixedTarget/batch/ganga.py diff --git a/fixedTarget/batch/ganga.py b/fixedTarget/batch/ganga.py new file mode 100644 index 0000000..708dad6 --- /dev/null +++ b/fixedTarget/batch/ganga.py @@ -0,0 +1,55 @@ +# ----------------------------------------------------- +# Put the functions defined here into your ~/.ganga.py +# and it will be available in the Ganga namespace. +# Then you can just do mergeOutput(jobNo) and it will +# submit the job. +# ----------------------------------------------------- + +def args_list_to_dict(args_list): + result = {} + i = 0 + while i < len(args_list): + key = str(args_list[i]).lstrip("-") + # Check if next item exists and is not another flag + if i + 1 < len(args_list) and isinstance(args_list[i + 1], str) and not args_list[i + 1].startswith("-"): + result[key] = args_list[i + 1] + i += 2 + else: + # Flag without value → set to True + result[key] = True + i += 1 + return result + +def mergeOutput(jNo): + + script_path = "/path/to/scripts/" # set this to wherever you checked out the scripts package + + j = Job(name = f"Merge output of production job {jNo}") + j.virtualization = Apptainer(image="/cvmfs/unpacked.cern.ch/registry.cern.ch/ship/gha-runner:latest/") + j.virtualization.mounts = {'/cvmfs':'/cvmfs'} + j.application = Executable(exe = File(script_path + 'wn_script.py')) + j.inputfiles = [LocalFile(script_path + "merger.py")] + + # Copy over the args but change the run file from run_fixedTarget.py to merger + j_arg_dict = args_list_to_dict(j.application.args) + this_site = j_arg_dict["site"] + myargs = ['merger.py' if _a=='run_fixedTarget.py' else _a for _a in jobs(jNo).application.args] + myargs.extend(['-j', jNo, '--prodSite', this_site, '--useLocalFile']) + myargs.append('-i') + + # Now grab the files + if this_site == "GRIDPP": # Do something different for GRIDPP where the files are on Dirac + inputfiles = jobs(jNo).backend.getOutputDataAccessURLs(protocol='"root"') + else: + inputfiles = [] + for sj in jobs(jNo).subjobs.select(status="completed"): + for _f in sj.outputfiles: + if isinstance(_f, MassStorageFile) + inputfiles.append(_f.locations[0]) + myargs.extend(inputfiles) + j.application.args = myargs + + j.outputfiles = [LocalFile('pythia_*.root')] + j.backend = Condor() + j.backend.cdf_options["+MaxRuntime"] = '3000' # Set this to something reasonable + j.submit() diff --git a/fixedTarget/batch/wn_script.py b/fixedTarget/batch/wn_script.py index ab9899a..044fbd4 100755 --- a/fixedTarget/batch/wn_script.py +++ b/fixedTarget/batch/wn_script.py @@ -47,6 +47,14 @@ def main(): default = "run_fixedTarget.py" ) + parser.add_argument( + "--useLocalFile", + help="Use a file on the WN, rather than defaulting to FairShip/macro", + default=False, + action="store_true", + type=bool + ) + # Everything after this is passed through to the FairShip script parser.add_argument( "script_args", @@ -60,8 +68,9 @@ def main(): FS_INSTALL = args.fs_install or '/cvmfs/ship.cern.ch/' + args.cvmfs_version WORK_DIR = args.work_dir or f"{FS_INSTALL}/sw/" INIT_SCRIPT = args.init_script or f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/etc/profile.d/init.sh" -# RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" - RUN_SCRIPT=args.runfile + RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" + if args.useLocalFile: + RUN_SCRIPT=args.runfile fs_version = args.cvmfs_version fs_tag = args.FairShip_tag or args.cvmfs_version @@ -80,7 +89,6 @@ def main(): command = f""" export WORK_DIR="{WORK_DIR}" source "{INIT_SCRIPT}" - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd) python "{RUN_SCRIPT}" {passthrough} """ From 5c0cdd99a080c568c278cb2b0647a1d8cd6d13dd Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Tue, 7 Apr 2026 09:59:14 +0100 Subject: [PATCH 3/9] feat: pythonise (#7) Essentially, try and hide all the metadata in the job application args, so that the postprocessor reads everything from the job, rather than having to hard code the info. To help this, switch the WN run script to be a python file with ArgParser instead of a bash script. --- fixedTarget/batch/gangaScript.py | 8 +- fixedTarget/batch/postprocessor_master.py | 30 ++++++-- fixedTarget/batch/wn_script.py | 93 +++++++++++++++++++++++ 3 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 fixedTarget/batch/wn_script.py diff --git a/fixedTarget/batch/gangaScript.py b/fixedTarget/batch/gangaScript.py index 4157898..f057d60 100644 --- a/fixedTarget/batch/gangaScript.py +++ b/fixedTarget/batch/gangaScript.py @@ -3,11 +3,9 @@ import random SITE = 'CERN' # or GENT or GRIDPP -os.environ["SITE"] = SITE # Set this path to wherever you want the output to go config['Output']['MassStorageFile']['uploadOptions']['path'] = '/eos/experiment/ship/simulation/bkg/Mbias2026/CERN' config['Output']['MassStorageFile']['uploadOptions']['defaultProtocol'] = 'root://eospublic.cern.ch//eos/experiment/ship/simulation/bkg/Mbias2026/CERN' - # Now set up the random seed, how many events per subjobs and how many events total user = os.environ.get("USER") random.seed(user + str(time.time())) @@ -31,7 +29,7 @@ for J in range(nJ): j = Job(name = f'run fixed target production number {J} - {nSJ * evtsPerJob} events') - j.application = Executable(exe = File('bashScript.sh'), args = ['-o', '"./"', '-n', evtsPerJob, '-e', str(ecut)]) + j.application = Executable(exe = File('wn_script.py'), args = ['--runfile', 'run_fixedTarget.py', '--cvmfs_version', '26.03', '--site', SITE, '--', '-o', '"./"', '-n', evtsPerJob, '-e', str(ecut)]) # IMPORTANT: Only put the run seed in the splitter arguments j.splitter = ArgSplitter(args = [['-r', startRun + J * nSJ + _i] for _i in range(nSJ)], append = True) @@ -46,8 +44,8 @@ # Add in the postprocessor to do the file registration # cc = CustomChecker(module = 'postprocessor.py') - cc = CustomChecker(module = 'postprocessor_master.py') - fc = FileChecker(files = ['stdout'], searchStrings = ['Macro finished successfully.'], failIfFound = False) + cc = CustomChecker(module = 'postprocessor_master.py', checkSubjobs=False) + fc = FileChecker(files = ['stdout'], searchStrings = ['Macro finished successfully.'], failIfFound = False, checkMaster=False) j.postprocessors.append(fc) j.postprocessors.append(cc) j.comment = f'{evtsPerJob} events in each of {nSJ} subjobs' diff --git a/fixedTarget/batch/postprocessor_master.py b/fixedTarget/batch/postprocessor_master.py index 94b65e0..5cb8e53 100644 --- a/fixedTarget/batch/postprocessor_master.py +++ b/fixedTarget/batch/postprocessor_master.py @@ -2,7 +2,20 @@ import rucio_it_tools.rucio_it_register import os -SITE = os.environ.get("SITE") +def args_list_to_dict(args_list): + result = {} + i = 0 + while i < len(args_list): + key = str(args_list[i]).lstrip("-") + # Check if next item exists and is not another flag + if i + 1 < len(args_list) and isinstance(args_list[i + 1], str) and not args_list[i + 1].startswith("-"): + result[key] = args_list[i + 1] + i += 2 + else: + # Flag without value → set to True + result[key] = True + i += 1 + return result # This is a config file set up for SHiP if 'RUCIO_CONFIG' not in os.environ: @@ -13,10 +26,15 @@ def check(j): if j.master: return True file_list = [] + j_arg_dict = args_list_to_dict(j.application.args) + if 'FairShip_tag' not in j_arg_dict.keys(): + j_arg_dict['FairShip_tag'] = j_arg_dict['cvmfs_version'] + for _sj in j.subjobs: if not _sj.status in ['completed', 'completing']: print(f"WARNING: Subjobs {_sj.id} did not complete") continue + sj_arg_dict = args_list_to_dict(_sj.application.args) for _f in _sj.outputfiles: if isinstance(_f, MassStorageFile): _loc = _f.locations[0] @@ -38,16 +56,16 @@ def check(j): "adler32": _checksum }) metadata = { - "runfile": "run_fixedTarget.py", + "runfile": j_arg_dict['runfile'], "production_type": "target production", "job_args" : str([_a for _a in j.application.args]), "creator": os.environ.get("USER"), "ganga_id": str(j.id), - "FairShip_tag": "26.03", # mainly useful if using local version of FairShip - "cvmfs_version": "26.03", + "FairShip_tag": j_arg_dict['FairShip_tag'], # mainly useful if using local version of FairShip + "cvmfs_version": j_arg_dict['cvmfs_version'], "comment": j.comment, "data_type": "simulation", - "production_site": SITE, + "production_site": j_arg_dict['site'], "run_nos" : str([(_sj.id, _sj.application.args[-1]) for _sj in j.subjobs if _sj.status=='completed']), "title" : j.name, } @@ -60,7 +78,7 @@ def check(j): rse_name = "SHIP_TIER_0_DISK", files = file_list, metadata = metadata, - # dry_run = True +# dry_run = True ) except Exception as e: print(f"ERROR: Not able to register file {file_list} with rucio: {e}") diff --git a/fixedTarget/batch/wn_script.py b/fixedTarget/batch/wn_script.py new file mode 100644 index 0000000..4309003 --- /dev/null +++ b/fixedTarget/batch/wn_script.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import shlex + +def main(): + parser = argparse.ArgumentParser( + description="Run FairShip fixed target simulation with environment setup" + ) + + # Main configurable inputs + parser.add_argument( + "--fs-install", + help="Base FairShip installation path" + ) + + parser.add_argument( + "--cvmfs_version", + default='26.03', + help="Which CVMFS version of FairShip to use (default: 26.03)" + ) + + parser.add_argument( + "--FairShip_tag", + help="What tag is used. Defaults to the cvmfs version if not set" + ) + + parser.add_argument( + "--site", + default="CERN", + help="Which site are we running - for setting site-specific options" + ) + + parser.add_argument( + "--work-dir", + help="WORK_DIR (default: /sw/)" + ) + + parser.add_argument( + "--init-script", + help="Path to init.sh (default derived from fs-install)" + ) + + parser.add_argument( + "--runfile", + help="Which python file to run (taken from the install dir location)", + default = "run_fixedTarget.py" + ) + + # Everything after this is passed through to the FairShip script + parser.add_argument( + "script_args", + nargs=argparse.REMAINDER, + help="Arguments passed to runfile" + ) + + args = parser.parse_args() + + # Derive defaults if not provided + FS_INSTALL = args.fs_install or '/cvmfs/ship.cern.ch/' + args.cvmfs_version + WORK_DIR = args.work_dir or f"{FS_INSTALL}/sw/" + INIT_SCRIPT = args.init_script or f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/etc/profile.d/init.sh" + RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" + + fs_version = args.cvmfs_version + fs_tag = args.FairShip_tag or args.cvmfs_version + print(f"INFO: Running at the site {args.site}") + print(f"INFO: Running with FairShip CVMFS version {fs_version} and tag {fs_tag}") + print(f"INFO: Environment set up for FairShip located at {FS_INSTALL}") + + + # Safely quote passthrough args + script_args = args.script_args[1:] if args.script_args[:1] == ["--"] else args.script_args + passthrough = " ".join(shlex.quote(a) for a in script_args) + + print(f"INFO: Executing: python {RUN_SCRIPT} {passthrough}") + + # Build bash command + command = f""" + export WORK_DIR="{WORK_DIR}" + source "{INIT_SCRIPT}" + python "{RUN_SCRIPT}" {passthrough} + """ + + subprocess.run(["bash", "-c", command], check=True) + + print("INFO: Finished running. These files are on the WN:") + + subprocess.run(["ls", "-lh"]) + + +if __name__ == "__main__": + main() From f0795affff127ab6b4bb53d2cb9bb8ae45b1c8a6 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Tue, 7 Apr 2026 12:09:28 +0100 Subject: [PATCH 4/9] fixes --- fixedTarget/batch/ganga.py | 15 +++++++++------ fixedTarget/batch/merger.py | 2 +- fixedTarget/batch/wn_script.py | 3 +-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fixedTarget/batch/ganga.py b/fixedTarget/batch/ganga.py index 708dad6..06204bc 100644 --- a/fixedTarget/batch/ganga.py +++ b/fixedTarget/batch/ganga.py @@ -22,34 +22,37 @@ def args_list_to_dict(args_list): def mergeOutput(jNo): - script_path = "/path/to/scripts/" # set this to wherever you checked out the scripts package + script_path = "/vols/lhcb/masmith/SHiP/htcondor_submission_scripts/fixedTarget/batch/" # set this to wherever you checked out the scripts package j = Job(name = f"Merge output of production job {jNo}") j.virtualization = Apptainer(image="/cvmfs/unpacked.cern.ch/registry.cern.ch/ship/gha-runner:latest/") - j.virtualization.mounts = {'/cvmfs':'/cvmfs'} + j.virtualization.mounts = {'/cvmfs':'/cvmfs', '/home/hep':'/home/hep'} # Adjust for your own setup j.application = Executable(exe = File(script_path + 'wn_script.py')) j.inputfiles = [LocalFile(script_path + "merger.py")] # Copy over the args but change the run file from run_fixedTarget.py to merger - j_arg_dict = args_list_to_dict(j.application.args) + j_arg_dict = args_list_to_dict(jobs(jNo).application.args) this_site = j_arg_dict["site"] myargs = ['merger.py' if _a=='run_fixedTarget.py' else _a for _a in jobs(jNo).application.args] - myargs.extend(['-j', jNo, '--prodSite', this_site, '--useLocalFile']) + myargs.insert(0,"--useLocalFile") + myargs.extend(['-j', jNo, '--prodSite', this_site]) myargs.append('-i') # Now grab the files if this_site == "GRIDPP": # Do something different for GRIDPP where the files are on Dirac inputfiles = jobs(jNo).backend.getOutputDataAccessURLs(protocol='"root"') + config["DIRAC"]["userVO"] = 'gridpp' + config["DIRAC"]["allDiracSE"] = ["UKI-LT2-IC-HEP-disk"] else: inputfiles = [] for sj in jobs(jNo).subjobs.select(status="completed"): for _f in sj.outputfiles: - if isinstance(_f, MassStorageFile) + if isinstance(_f, MassStorageFile): inputfiles.append(_f.locations[0]) myargs.extend(inputfiles) j.application.args = myargs - j.outputfiles = [LocalFile('pythia_*.root')] + j.outputfiles = [DiracFile('pythia8_*.root')] j.backend = Condor() j.backend.cdf_options["+MaxRuntime"] = '3000' # Set this to something reasonable j.submit() diff --git a/fixedTarget/batch/merger.py b/fixedTarget/batch/merger.py index d2aa37f..93d3c6d 100644 --- a/fixedTarget/batch/merger.py +++ b/fixedTarget/batch/merger.py @@ -62,7 +62,7 @@ def mergeFiles(myfiles, tmpFile): if not len(files_to_merge)>0: print("ERROR: no files to merge! Doing nothing") sys.exit() -print("INFO: Merging %s files from job %s" % (len(files_to_merge), arg.gangaJob)) +print("INFO: Merging %s files from job %s" % (len(files_to_merge), args.gangaJob)) total_pot = len(files_to_merge) * args.genEvents outName = f"pythia8_Geant4_eCut_{args.eCut}_PoT_{total_pot}_{args.site}_j{args.gangaJob}.root" diff --git a/fixedTarget/batch/wn_script.py b/fixedTarget/batch/wn_script.py index 044fbd4..34c391d 100755 --- a/fixedTarget/batch/wn_script.py +++ b/fixedTarget/batch/wn_script.py @@ -52,7 +52,6 @@ def main(): help="Use a file on the WN, rather than defaulting to FairShip/macro", default=False, action="store_true", - type=bool ) # Everything after this is passed through to the FairShip script @@ -69,7 +68,7 @@ def main(): WORK_DIR = args.work_dir or f"{FS_INSTALL}/sw/" INIT_SCRIPT = args.init_script or f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/etc/profile.d/init.sh" RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" - if args.useLocalFile: + if args.useLocalFile or True: RUN_SCRIPT=args.runfile fs_version = args.cvmfs_version From 841316c3f8010b9d9566fe73fba69250d709f164 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Tue, 7 Apr 2026 12:16:13 +0100 Subject: [PATCH 5/9] only parse known args --- fixedTarget/batch/merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fixedTarget/batch/merger.py b/fixedTarget/batch/merger.py index 93d3c6d..a8a999e 100644 --- a/fixedTarget/batch/merger.py +++ b/fixedTarget/batch/merger.py @@ -49,7 +49,7 @@ def mergeFiles(myfiles, tmpFile): help="Which site was this run at" ) -args = parser.parse_args() +args, _ = parser.parse_known_args() if args.prodSite == "GRIDPP": From ce04f1a7c2c87819a8578b7b7d6dcb2f46a552d0 Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Tue, 7 Apr 2026 16:57:41 +0100 Subject: [PATCH 6/9] Remove hardcoded X509_USER_PROXY path Removed hardcoded proxy path for GRIDPP environment. --- fixedTarget/batch/merger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fixedTarget/batch/merger.py b/fixedTarget/batch/merger.py index a8a999e..77b014b 100644 --- a/fixedTarget/batch/merger.py +++ b/fixedTarget/batch/merger.py @@ -53,7 +53,6 @@ def mergeFiles(myfiles, tmpFile): if args.prodSite == "GRIDPP": - os.environ["X509_USER_PROXY"]="/home/hep/mesmith/private/my_proxy.pem" os.environ["X509_VOMSES"]="/cvmfs/grid.cern.ch/etc/grid-security/vomses" os.environ["X509_VOMS_DIR"]="/cvmfs/grid.cern.ch/etc/grid-security/vomsdir" os.environ["X509_CERT_DIR"]="/cvmfs/grid.cern.ch/etc/grid-security/certificates" From 17d4b987b2d55ab721bf8d02bf582da8001b1803 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Tue, 7 Apr 2026 17:25:07 +0100 Subject: [PATCH 7/9] update --- fixedTarget/batch/merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fixedTarget/batch/merger.py b/fixedTarget/batch/merger.py index a8a999e..e0f23f1 100644 --- a/fixedTarget/batch/merger.py +++ b/fixedTarget/batch/merger.py @@ -65,7 +65,7 @@ def mergeFiles(myfiles, tmpFile): print("INFO: Merging %s files from job %s" % (len(files_to_merge), args.gangaJob)) total_pot = len(files_to_merge) * args.genEvents -outName = f"pythia8_Geant4_eCut_{args.eCut}_PoT_{total_pot}_{args.site}_j{args.gangaJob}.root" +outName = f"pythia8_Geant4_eCut_{args.eCut}_PoT_{total_pot}_{args.prodSite}_j{args.gangaJob}.root" mergeFiles(files_to_merge, outName) From 9d83369c491cf864509cdfe9d501f994b50b2fd5 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Tue, 7 Apr 2026 17:26:02 +0100 Subject: [PATCH 8/9] temp --- fixedTarget/batch/gangaScript.py | 35 ++++++++++++++++++-------------- fixedTarget/batch/wn_script.py | 2 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/fixedTarget/batch/gangaScript.py b/fixedTarget/batch/gangaScript.py index f057d60..8409596 100644 --- a/fixedTarget/batch/gangaScript.py +++ b/fixedTarget/batch/gangaScript.py @@ -2,10 +2,10 @@ import os import random -SITE = 'CERN' # or GENT or GRIDPP +SITE = 'GRIDPP' # or GENT or GRIDPP # Set this path to wherever you want the output to go -config['Output']['MassStorageFile']['uploadOptions']['path'] = '/eos/experiment/ship/simulation/bkg/Mbias2026/CERN' -config['Output']['MassStorageFile']['uploadOptions']['defaultProtocol'] = 'root://eospublic.cern.ch//eos/experiment/ship/simulation/bkg/Mbias2026/CERN' +#config['Output']['MassStorageFile']['uploadOptions']['path'] = '/eos/experiment/ship/simulation/bkg/Mbias2026/CERN' +#config['Output']['MassStorageFile']['uploadOptions']['defaultProtocol'] = 'root://eospublic.cern.ch//eos/experiment/ship/simulation/bkg/Mbias2026/CERN' # Now set up the random seed, how many events per subjobs and how many events total user = os.environ.get("USER") random.seed(user + str(time.time())) @@ -19,23 +19,28 @@ run_min = 600000000 run_max = 900000000 -evtsPerJob = 800000 # 200000 -nJ = 70 # in total want 14000 subjobs per week as a first try -nSJ = 200 # fixed number of subjobs per job to register all subjobs on rucio at the same time +evtsPerJob =10000# 500000# 800000 # 200000 +nJ = 1# 70 # in total want 14000 subjobs per week as a first try +nSJ = 1#400# 200 # fixed number of subjobs per job to register all subjobs on rucio at the same time ecut = 30 # 5 -startRun = random.randint(run_min, run_max - nJ * nSJ) +totalEvts = evtsPerJob * nSJ +startRun = random.randint(run_min, run_max - nJ * nSJ) +#docker pull registry.cern.ch/ship/gha-runner@sha256:2727d9b945b8b41554e3a9f8b03cb028f84b36464994470a4dfb9aec599c5689 for J in range(nJ): j = Job(name = f'run fixed target production number {J} - {nSJ * evtsPerJob} events') - j.application = Executable(exe = File('wn_script.py'), args = ['--runfile', 'run_fixedTarget.py', '--cvmfs_version', '26.03', '--site', SITE, '--', '-o', '"./"', '-n', evtsPerJob, '-e', str(ecut)]) - + j.virtualization = Apptainer(image="/cvmfs/unpacked.cern.ch/registry.cern.ch/ship/gha-runner:latest/") + j.virtualization.mounts = {'/cvmfs':'/cvmfs'} + j.application = Executable(exe = File('wn_script.py'), args = ['--runfile', 'run_fixedTarget.py', '--cvmfs_version', '26.03', '--site', SITE, '--', '-o', './', '-n', evtsPerJob, '-e', str(ecut)]) # IMPORTANT: Only put the run seed in the splitter arguments j.splitter = ArgSplitter(args = [['-r', startRun + J * nSJ + _i] for _i in range(nSJ)], append = True) - j.outputfiles = [MassStorageFile('pythia8_evtgen_Geant4_*.root')] - j.backend = Condor() - j.backend.cdf_options['+MaxRuntime'] = '86000' + j.outputfiles = [DiracFile('pythia8_evtgen_Geant4_*.root')] + j.backend = Dirac() + j.backend.settings['CPUTime'] = '40000' +# j.backend = Condor() +# j.backend.cdf_options['+MaxRuntime'] = '86000' # For running at CERN only if SITE == 'CERN': @@ -44,9 +49,9 @@ # Add in the postprocessor to do the file registration # cc = CustomChecker(module = 'postprocessor.py') - cc = CustomChecker(module = 'postprocessor_master.py', checkSubjobs=False) +# cc = CustomChecker(module = 'postprocessor_master.py', checkSubjobs=False) fc = FileChecker(files = ['stdout'], searchStrings = ['Macro finished successfully.'], failIfFound = False, checkMaster=False) j.postprocessors.append(fc) - j.postprocessors.append(cc) - j.comment = f'{evtsPerJob} events in each of {nSJ} subjobs' +# j.postprocessors.append(cc) + j.comment = f'%s events in each of %s subjobs, %.2f million total' % (evtsPerJob, nSJ, totalEvts/1.e6) j.submit() diff --git a/fixedTarget/batch/wn_script.py b/fixedTarget/batch/wn_script.py index 34c391d..6bde7c4 100755 --- a/fixedTarget/batch/wn_script.py +++ b/fixedTarget/batch/wn_script.py @@ -68,7 +68,7 @@ def main(): WORK_DIR = args.work_dir or f"{FS_INSTALL}/sw/" INIT_SCRIPT = args.init_script or f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/etc/profile.d/init.sh" RUN_SCRIPT = f"{FS_INSTALL}/sw/slc9_x86-64/FairShip/latest/macro/{args.runfile}" - if args.useLocalFile or True: + if args.useLocalFile: RUN_SCRIPT=args.runfile fs_version = args.cvmfs_version From fe01394f96b7b32951a8eeba516e6ee6dca253c9 Mon Sep 17 00:00:00 2001 From: Mark E Smith Date: Tue, 7 Apr 2026 17:32:06 +0100 Subject: [PATCH 9/9] tidy --- fixedTarget/batch/gangaScript.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/fixedTarget/batch/gangaScript.py b/fixedTarget/batch/gangaScript.py index 322fa61..37c60d7 100644 --- a/fixedTarget/batch/gangaScript.py +++ b/fixedTarget/batch/gangaScript.py @@ -36,10 +36,12 @@ # IMPORTANT: Only put the run seed in the splitter arguments j.splitter = ArgSplitter(args = [['-r', startRun + J * nSJ + _i] for _i in range(nSJ)], append = True) j.outputfiles = [DiracFile('pythia8_evtgen_Geant4_*.root')] - j.backend = Dirac() - j.backend.settings['CPUTime'] = '40000' -# j.backend = Condor() -# j.backend.cdf_options['+MaxRuntime'] = '86000' + if SITE == 'GRIDPP': + j.backend = Dirac() + j.backend.settings['CPUTime'] = '40000' + else: + j.backend = Condor() + j.backend.cdf_options['+MaxRuntime'] = '86000' # For running at CERN only if SITE == 'CERN': @@ -47,10 +49,10 @@ j.backend.cdf_options['accounting_group'] = 'group_u_SHIP.u_ship_cg' # Add in the postprocessor to do the file registration - # cc = CustomChecker(module = 'postprocessor.py') -# cc = CustomChecker(module = 'postprocessor_master.py', checkSubjobs=False) fc = FileChecker(files = ['stdout'], searchStrings = ['Macro finished successfully.'], failIfFound = False, checkMaster=False) j.postprocessors.append(fc) -# j.postprocessors.append(cc) + if SITE == 'CERN': + cc = CustomChecker(module = 'postprocessor_master.py', checkSubjobs=False) + j.postprocessors.append(cc) j.comment = f'%s events in each of %s subjobs, %.2f million total' % (evtsPerJob, nSJ, totalEvts/1.e6) j.submit()