diff --git a/src/scripts/production/python/README.md b/src/scripts/production/python/README.md index e9566bae5..0ae355ba3 100644 --- a/src/scripts/production/python/README.md +++ b/src/scripts/production/python/README.md @@ -8,6 +8,7 @@ other scripts can be easily added. Does not support neutrino scattering event ge - `vAScatteringGridSubmitter.py`: submits the jobs to create splines and generate nu-A events other scripts can be easily added. Does not support electron scattering event generation +NOTE: avoid using the scripts as a black box. Job lifetime and memory should be optimized for each job. Runing with the default might bring your jobs to hold. ## Requirements - You must have the $GENIE enviromental variable to your GENIE-MC/Generator directory. This is required only to run the python scripts, no previous compilation of the GENIE-MC/Generator code is required. diff --git a/src/scripts/production/python/XMLGridSubmitter.py b/src/scripts/production/python/XMLGridSubmitter.py index 687bfcea9..3f7c624fa 100644 --- a/src/scripts/production/python/XMLGridSubmitter.py +++ b/src/scripts/production/python/XMLGridSubmitter.py @@ -170,7 +170,7 @@ if opts.MotherDir !='' : vNMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vN/' - command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,opts.vNList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) if loop_i == 2 : @@ -184,7 +184,7 @@ if opts.MotherDir !='' : vAMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vA/' - command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,opts.vAList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) loop_i += 1 diff --git a/src/scripts/production/python/eAScatteringGridSubmitter.py b/src/scripts/production/python/eAScatteringGridSubmitter.py index 7ab2e1f41..08dd8c7de 100644 --- a/src/scripts/production/python/eAScatteringGridSubmitter.py +++ b/src/scripts/production/python/eAScatteringGridSubmitter.py @@ -38,6 +38,7 @@ op.add_option("--genie-topdir", dest="GENIE", default=os.getenv('GENIE'), help = "GENIE topdir: %default") op.add_option("--jobs-topdir", dest="JOBSTD", default=os.getenv('PWD'), help="Top level dir for the job files (default: %default)") op.add_option("--source-prod-dir", dest="MotherDir", default='', help="Jobs topdir used as a source for missing xsec splines.") +op.add_option("--total-xsec", dest="XSEC", default='', help="Total cross section file to use for event production") op.add_option("--config-dir", dest="CONF", default='', help="Path to GENIE config dir") op.add_option("--probe-list", dest="PROBELIST", default='11', help = "Comma separated list of lepton flavour (neutrino and electrons are handled). Default: %default.") op.add_option("--nu-tgt-list", dest="NUTGTLIST", default='all', help = "Comma separated list of Targets. Default: %default.") @@ -51,8 +52,10 @@ op.add_option("--event-generator-list", dest="EvGenList", default='EM', help="Event generator list to be used for event generation. Default EM") op.add_option("--nu-ntotevents", dest="NuEvents", type="int", default=10000, help="Number of total events, default: 100 k") op.add_option("--e-ntotevents", dest="EEvents", type="int", default=100000, help="Number of total events, default: 100 k") -op.add_option("--nmaxevents",dest="NMax", type="int", default=100000,help="Max number of events to run per event generation, default 400k") +op.add_option("--nmaxevents",dest="NMax", type="int", default=400000,help="Max number of events to run per event generation, default %default") op.add_option("--ebeam-energy", dest="EnergyBeam", default="2", help="Comma separated list of beam energy for electrons. Default %default GeV") +op.add_option("--gst-output", dest="GSTOutput", default=False, action="store_true",help="Store gst root file.") +op.add_option("--no-ghep-output", dest="NoGHEPOutput", default=False, action="store_true",help="GHEP GENIE files is removed to reduce memory.") op.add_option("--starting-point", dest="start_ID", type="int", default=0, help="0 -> Free nucleon splines, 1 -> combine free nucl splines, 2 -> Compound nuclei splines, 3 -> Combine compound nuclei splines, 4 -> Event Production") op.add_option("--stopping-point", dest="end_ID", type="int", default=9999, help="Numbers as above, Default: 9999") op.add_option("--tune", dest="TUNE", default="G18_02a_02_11b", help="Tune to be compared against data (default: %default)") @@ -62,6 +65,8 @@ op.add_option("--job-lifetime-vA", dest="vAJOBLIFE", default=3, help="Expected lifetime on the grid for all the vA spline jobs to be finished, default %default h") op.add_option("--job-lifetime-generation", dest="GENJOBLIFE", default=4, help="Expected lifetime on the grid for all the event generation jobs to be finished, default %default h") op.add_option("--job-lifetime-group", dest="GROUPJOBLIFE", default=1, help="Expected lifetime on the grid for all the grouping jobs to be finished, default 1h") +op.add_option("--job-memory-generation", dest="GENJOBMEM", default=2, help="Expected memory usage for each event generation job, default %default GB") +op.add_option("--job-disk-generation", dest="GENJOBDISK", default=2000, help="Expected disk space for each event generation job, default %default MB") op.add_option("--store-comitinfo", dest="STORECOMMIT", default=False, action="store_true", help="Store command line in jobstopdir directory") opts, args = op.parse_args() @@ -135,6 +140,12 @@ vAdir = opts.JOBSTD+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vA/' vAsplines = vAdir+'total_xsec.xml' +if opts.XSEC : + if os.path.isfile(opts.XSEC) == False : + print(" Input XSec file doesn't exist ") + exit() + vAsplines = opts.XSEC + # configure setup if opts.GRID == 'FNAL' : setup_file = opts.GENIE+'/src/scripts/production/python/setup_FNAL.sh' @@ -178,8 +189,8 @@ vNMotherDir = '' if opts.MotherDir !='' : vNMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vN/' - - command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + + command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,opts.vNList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) if loop_i == 2 : @@ -192,13 +203,12 @@ vAMotherDir = '' if opts.MotherDir !='' : vAMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vA/' - - command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,opts.vAList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) if loop_i == 4 : # ID = 4 # Event generation commands - command_dict.update( eA.eScatteringGenCommands(opts.PROBELIST,opts.ETGTLIST,opts.EnergyBeam,vAsplines,opts.EEvents,opts.TUNE, opts.EvGenList, opts.NMax,version,opts.CONF, opts.ARCH, opts.PROD, opts.CYCLE,opts.GRID, opts.GROUP,opts.SOFTW,opts.GENIE,opts.JOBSTD,grid_setup,genie_setup,opts.GENJOBLIFE,opts.BRANCH,opts.GIT_LOCATION) ) + command_dict.update( eA.eScatteringGenCommands(opts.PROBELIST,opts.ETGTLIST,opts.EnergyBeam,vAsplines,opts.EEvents,opts.TUNE, opts.EvGenList, opts.NMax, opts.GSTOutput, opts.NoGHEPOutput,version,opts.CONF, opts.ARCH, opts.PROD, opts.CYCLE,opts.GRID, opts.GROUP,opts.SOFTW,opts.GENIE,opts.JOBSTD,grid_setup,genie_setup,opts.GENJOBLIFE,opts.GENJOBMEM,opts.GENJOBDISK,opts.BRANCH,opts.GIT_LOCATION) ) total_time += int(opts.GENJOBLIFE) loop_i += 1 diff --git a/src/scripts/production/python/event_generation/eScatteringGenCommands.py b/src/scripts/production/python/event_generation/eScatteringGenCommands.py index 0eae7e4b0..cb93295d3 100644 --- a/src/scripts/production/python/event_generation/eScatteringGenCommands.py +++ b/src/scripts/production/python/event_generation/eScatteringGenCommands.py @@ -28,18 +28,17 @@ '1000180400', '1000200400', '1000200480', '1000260560', '1000791970', '1000822080', '1000922380'] def eScatteringGenCommands( e_list = "11",tgt_list="1000060120", EBeam_list="2", xspl_file="total_xsec.xml",ntotevents=1000000, - tune='G18_02_02_11b',gen_list="EM", nmaxrun=100000, version='master', conf_dir='', arch='SL6.x86_64', + tune='G18_02_02_11b',gen_list="EM", nmaxrun=100000, gst_output=False, no_ghep=False,version='master', conf_dir='', arch='SL6.x86_64', production='routine_validation', cycle='01', grid_system='FNAL', group='genie', softw_topdir=os.getenv('GENIE_MASTER_DIR'), genie_topdir=os.getenv('GENIE'), jobs_topdir=os.getenv('PWD'), grid_setup = os.getenv('GENIE')+'src/scripts/production/python/setup_FNAL.sh', - genie_setup= os.getenv('GENIE')+'src/scripts/production/python/setup_GENIE.sh', time='10', git_branch = "master", git_loc="https://github.com/GENIE-MC/Generator") : + genie_setup= os.getenv('GENIE')+'src/scripts/production/python/setup_GENIE.sh', time='10', memory='1', disk='2000',git_branch = "master", git_loc="https://github.com/GENIE-MC/Generator") : jobs_dir = jobs_topdir+'/'+version+'-'+production+'_'+cycle+'-eScattering' # Make directory if not os.path.exists(jobs_dir) : os.mkdir(jobs_dir) - # Electron list final_e_list = [] if e_list != 'all': @@ -67,10 +66,12 @@ def eScatteringGenCommands( e_list = "11",tgt_list="1000060120", EBeam_list="2", if not isinstance(nsubruns, int) : nsubruns = 1+int(nsubruns) + xsec_filename = os.path.basename(xspl_file) + if grid_system == 'FNAL' : - input_xsec = "\$CONDOR_DIR_INPUT/total_xsec.xml" + input_xsec = "\$CONDOR_DIR_INPUT/"+xsec_filename else : - input_xsec = free_nuc_dir+"/total_xsec.xml" + input_xsec = free_nuc_dir+xsec_filename command_list = [] for e in final_e_list : @@ -88,12 +89,19 @@ def eScatteringGenCommands( e_list = "11",tgt_list="1000060120", EBeam_list="2", jobname = "e_on_"+str(tgt)+"_"+str(int((float(E)*1000)))+"MeV_"+str(isubrun) evgen_command = "gevgen -p "+str(e)+" -n "+str(nev)+" -e "+E+" -t "+str(tgt)+" -r "+curr_subrune+" --seed "+str(curr_seed) - evgen_command += " --cross-sections "+input_xsec+" --event-generator-list "+gen_list+" --tune "+tune + " -o "+jobname+".ghep.root" + evgen_command += " --cross-sections "+input_xsec+" --event-generator-list "+gen_list+" --tune "+tune + " -o "+jobname+".ghep.root " + + out_files = [str(jobname+".ghep.root")] + if gst_output : + evgen_command += " ; gntpc -i "+jobname+".ghep.root -o "+jobname+".gst.root -f gst " + out_files.append(str(jobname+".gst.root")) + if no_ghep : + out_files = [str(jobname+".gst.root")] shell_file = '' if grid_system == 'FNAL' : - shell_file= FNAL.CreateShellScript ( evgen_command , jobs_dir, jobname, str(jobname+".ghep.root"), grid_setup, genie_setup, conf_dir, str(xspl_file), git_branch, git_loc ) - grid_command_options = FNAL.FNALShellCommands(grid_setup, genie_setup,time) + shell_file= FNAL.CreateShellScript ( evgen_command , jobs_dir, jobname, out_files, grid_setup, genie_setup, conf_dir, str(xspl_file), git_branch, git_loc ) + grid_command_options = FNAL.FNALShellCommands(grid_setup, genie_setup,time,memory,disk) command_list.append( "jobsub_submit "+grid_command_options+ " file://"+shell_file ) ## Add command list to dictionary; Key is 4 => event production diff --git a/src/scripts/production/python/event_generation/nuScatteringGenCommands.py b/src/scripts/production/python/event_generation/nuScatteringGenCommands.py index d0d511ee7..20799b2b3 100644 --- a/src/scripts/production/python/event_generation/nuScatteringGenCommands.py +++ b/src/scripts/production/python/event_generation/nuScatteringGenCommands.py @@ -39,7 +39,7 @@ '1000180400', '1000200400', '1000200480', '1000260560', '1000791970', '1000822080', '1000922380'] def nuScatteringGenCommands( nu_list = "14",tgt_mix="", EFlux_min=0, EFlux_max=100, flux="\'1/x\'", xspl_file="total_xsec.xml",ntotevents=1000000, - tune='G18_02_02_11b',gen_list="all", expname="general",nmaxrun=100000, version='master', conf_dir='', arch='SL6.x86_64', + tune='G18_02_02_11b',gen_list="all", expname="general",nmaxrun=100000, gst_output=False, no_ghep=False,version='master', conf_dir='', arch='SL6.x86_64', production='routine_validation', cycle='01', grid_system='FNAL', group='genie', softw_topdir=os.getenv('GENIE_MASTER_DIR'), genie_topdir=os.getenv('GENIE'), jobs_topdir=os.getenv('PWD'), grid_setup = os.getenv('GENIE')+'src/scripts/production/python/setup_FNAL.sh', @@ -109,11 +109,19 @@ def nuScatteringGenCommands( nu_list = "14",tgt_mix="", EFlux_min=0, EFlux_max=1 jobname = "nu_"+expname+"_"+str(isubrun) evgen_command = "gevgen -p "+str(nu)+" -n "+str(nev)+" -e "+EFlux_min+","+EFlux_max+" -f " +flux+" -t "+str(tgt_mix)+" -r "+curr_subrune+" --seed "+str(curr_seed) evgen_command += " --cross-sections "+input_xsec+" --tune "+tune + " -o "+jobname+".ghep.root" + if gen_list is not "all" : evgen_command += " --event-generator-list "+gen_list+" " shell_file = '' + + out_files = [str(jobname+".ghep.root")] + if gst_output : + evgen_command += " ; gntpc -i "+jobname+".ghep.root -o "+jobname+".gst.root -f gst " + out_files.append(str(jobname+".gst.root")) + if no_ghep : + out_files = [str(jobname+".gst.root")] if grid_system == 'FNAL' : - shell_file= FNAL.CreateShellScript ( evgen_command , jobs_dir, jobname, str(jobname+".ghep.root"), grid_setup, genie_setup, conf_dir, in_file_list, git_branch, git_loc ) + shell_file= FNAL.CreateShellScript ( evgen_command , jobs_dir, jobname, out_files, grid_setup, genie_setup, conf_dir, in_file_list, git_branch, git_loc ) grid_command_options = FNAL.FNALShellCommands(grid_setup, genie_setup,time) command_list.append( "jobsub_submit "+grid_command_options+ " file://"+shell_file ) diff --git a/src/scripts/production/python/setup_GENIE.sh b/src/scripts/production/python/setup_GENIE.sh index c7c499ae3..b629babf1 100644 --- a/src/scripts/production/python/setup_GENIE.sh +++ b/src/scripts/production/python/setup_GENIE.sh @@ -79,4 +79,4 @@ fi #--with-boost-inc=${BOOST_FQ_DIR}/include \ #--with-boost-lib=${BOOST_FQ_DIR}/lib -make +make -j4 diff --git a/src/scripts/production/python/vAScatteringGridSubmitter.py b/src/scripts/production/python/vAScatteringGridSubmitter.py index c5874d5c8..ec332fa24 100644 --- a/src/scripts/production/python/vAScatteringGridSubmitter.py +++ b/src/scripts/production/python/vAScatteringGridSubmitter.py @@ -39,6 +39,7 @@ op.add_option("--genie-topdir", dest="GENIE", default=os.getenv('GENIE'), help = "GENIE topdir: %default") op.add_option("--jobs-topdir", dest="JOBSTD", default=os.getenv('PWD'), help="Top level dir for the job files (default: %default)") op.add_option("--source-prod-dir", dest="MotherDir", default='', help="Jobs topdir used as a source for missing xsec splines.") +op.add_option("--total-xsec", dest="XSEC", default='', help="Total cross section file to use for event production") op.add_option("--config-dir", dest="CONF", default='', help="Path to GENIE config dir") op.add_option("--probe-list", dest="PROBELIST", default='14', help = "Comma separated list of lepton flavour (neutrino and electrons are handled). Default: %default.") op.add_option("--nu-tgt-list", dest="NUTGTLIST", default='all', help = "Comma separated list of Targets. Default: %default.") @@ -59,6 +60,8 @@ op.add_option("--e-beamenergy-list", dest="BEnergy", default="2", help="Electron beam energy" ) op.add_option("--flux", dest="FLUX", default="\'1/x\'", help="Neutrino flux. Default 1/x. To use an input root file, specify the location of the file and the TH1D name as follows: file.root,th1d_name") op.add_option("--exp-name", dest="EXPNAME", default="general", help="Neutrino experiment name, i.e. DUNE, MINERvA, T2K, etc. It is only used to tag the output files. Default: %default") +op.add_option("--gst-output", dest="GSTOutput", default=False, action="store_true",help="Store gst root file.") +op.add_option("--no-ghep-output", dest="NoGHEPOutput", default=False, action="store_true",help="GHEP GENIE files is removed to reduce memory.") op.add_option("--starting-point", dest="start_ID", type="int", default=0, help="0 -> Free nucleon splines, 1 -> combine free nucl splines, 2 -> Compound nuclei splines, 3 -> Combine compound nuclei splines, 4 -> Event Production") op.add_option("--stopping-point", dest="end_ID", type="int", default=9999, help="Numbers as above, Default: 9999") op.add_option("--tune", dest="TUNE", default="G18_02a_02_11b", help="Tune to be compared against data (default: %default)") @@ -143,6 +146,12 @@ vAdir = opts.JOBSTD+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vA/' vAsplines = vAdir+'total_xsec.xml' +if opts.XSEC : + if os.path.isfile(opts.XSEC) == False : + print(" Input XSec file doesn't exist ") + exit() + vAsplines = opts.XSEC + # configure setup if opts.GRID == 'FNAL' : setup_file = opts.GENIE+'/src/scripts/production/python/setup_FNAL.sh' @@ -187,7 +196,7 @@ if opts.MotherDir !='' : vNMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vN/' - command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + command_dict.update( group.GroupSplineCommands( True,vNdir,vNMotherDir,opts.TUNE,opts.vNList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False, opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) if loop_i == 2 : @@ -201,13 +210,13 @@ if opts.MotherDir !='' : vAMotherDir = opts.MotherDir+'/'+version+'-'+opts.PROD+'_'+opts.CYCLE+'-xsec_vA/' - command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) + command_dict.update( group.GroupSplineCommands( False,vAdir,vAMotherDir,opts.TUNE,opts.vAList,version,opts.CONF,opts.GRID,opts.GROUP,opts.ARCH,opts.PROD,opts.CYCLE,opts.SOFTW,opts.GENIE,grid_setup,genie_setup,opts.JOBSTD,False, False,opts.GROUPJOBLIFE,opts.BRANCH,opts.GIT_LOCATION ) ) total_time += int(opts.GROUPJOBLIFE) if loop_i == 4 : # ID = 4 # Event generation commands # Submit neutrino jobs - command_dict.update( nuA.nuScatteringGenCommands(opts.PROBELIST,opts.TGTMIX,opts.MinEnergyFlux,opts.MaxEnergyFlux,opts.FLUX,vAsplines,opts.NuEvents,opts.TUNE, opts.EvGenList, opts.EXPNAME, opts.NMax,version,opts.CONF, opts.ARCH, opts.PROD, opts.CYCLE,opts.GRID, opts.GROUP,opts.SOFTW,opts.GENIE,opts.JOBSTD,grid_setup,genie_setup,opts.GENJOBLIFE,opts.BRANCH,opts.GIT_LOCATION) ) + command_dict.update( nuA.nuScatteringGenCommands(opts.PROBELIST,opts.TGTMIX,opts.MinEnergyFlux,opts.MaxEnergyFlux,opts.FLUX,vAsplines,opts.NuEvents,opts.TUNE,opts.EvGenList,opts.EXPNAME,opts.NMax,opts.GSTOutput,opts.NoGHEPOutput,version,opts.CONF, opts.ARCH, opts.PROD, opts.CYCLE,opts.GRID, opts.GROUP,opts.SOFTW,opts.GENIE,opts.JOBSTD,grid_setup,genie_setup,opts.GENJOBLIFE,opts.BRANCH,opts.GIT_LOCATION) ) # Submit electron jobs # command_dict.update( eA.eScatteringGenCommands(opts.PROBELIST,opts.ETGTLIST,opts.BEnergy,vAsplines,opts.EEvents,opts.TUNE, opts.EvGenList, opts.NMax,version,opts.CONF, opts.ARCH, opts.PROD, opts.CYCLE,opts.GRID, opts.GROUP,opts.SOFTW,opts.GENIE,opts.JOBSTD,grid_setup,genie_setup,opts.GENJOBLIFE,opts.BRANCH,opts.GIT_LOCATION) ) total_time += int(opts.GENJOBLIFE) diff --git a/src/scripts/production/python/xsec_splines/FNALGridUtils.py b/src/scripts/production/python/xsec_splines/FNALGridUtils.py index 5fbb38650..fc1d6c7b0 100644 --- a/src/scripts/production/python/xsec_splines/FNALGridUtils.py +++ b/src/scripts/production/python/xsec_splines/FNALGridUtils.py @@ -41,7 +41,7 @@ def CreateShellScript ( commands , jobs_dir, shell_name, out_files, grid_setup, script.close() return shell_file -def FNALShellCommands(grid_setup, genie_setup, hours = 10, memory=1, disk=1, GraceMemory=4096, GraceLifeTime=6000): +def FNALShellCommands(grid_setup, genie_setup, hours = 10, memory=1, disk=500, GraceMemory=4096, GraceLifeTime=6000): grid_command_options = " -n --memory="+str(memory)+"GB --disk="+str(disk)+"MB --expected-lifetime="+str(hours)+"h " grid_command_options += " --OS=SL7 --lines '+FERMIHTC_AutoRelease=True' -f "+grid_setup+" -f "+genie_setup grid_command_options += " --lines '+FERMIHTC_GraceMemory="+str(GraceMemory)+"' --lines '+FERMIHTC_GraceLifetime="+str(GraceLifeTime)+"' --mail_on_error " @@ -63,6 +63,8 @@ def WriteXMLFile(commands_dict, start, end, jobs_dir, file_name='grid_submission else : command_list_next = command_list + if len(command_list) == 0 : + continue if len(command_list) == 1 : # serial if in_serial == False: script.write("\n") @@ -86,7 +88,6 @@ def WriteXMLFile(commands_dict, start, end, jobs_dir, file_name='grid_submission def WriteMainSubmissionFile(jobs_dir, genie_topdir, group, grid_setup='/src/scripts/production/python/setup_FNAL.sh', genie_setup='/src/scripts/production/python/setup_GENIE.sh', in_file_name='grid_submission.xml', expectedlife=60, memory=1, disk=500, out_file_name='fnal_dag_submit.fnal', jobs=1, role="Analysis"): fnal_file = jobs_dir+"/"+out_file_name - tar_file = jobs_dir+"/FNALTarFile.fnal.gov.tgz" if os.path.exists(fnal_file): os.remove(fnal_file) @@ -94,6 +95,6 @@ def WriteMainSubmissionFile(jobs_dir, genie_topdir, group, grid_setup='/src/scri script.write("#!/bin/bash\n") script.write("source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups ;\n") script.write("setup fife_utils ;\n") - script.write("jobsub_submit -G "+group+" --OS=SL7 --memory="+str(memory)+"GB --disk="+str(disk)+"GB --expected-lifetime="+str(expectedlife)+"h -N "+str(jobs)+" --role="+role+" --tar_file_name "+tar_file+" --dag file://"+in_file_name+";" ) + script.write("jobsub_submit -G "+group+" --OS=SL7 --memory="+str(memory)+"GB --disk="+str(disk)+"MB --expected-lifetime="+str(expectedlife)+"h -N "+str(jobs)+" --role="+role+" --dag file://"+in_file_name+";" ) return fnal_file diff --git a/src/scripts/production/python/xsec_splines/GroupSplineCommands.py b/src/scripts/production/python/xsec_splines/GroupSplineCommands.py index 312282b5f..6f1b7839a 100644 --- a/src/scripts/production/python/xsec_splines/GroupSplineCommands.py +++ b/src/scripts/production/python/xsec_splines/GroupSplineCommands.py @@ -50,26 +50,38 @@ -11: 'ebar' } -def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='', tune='G18_02_02_11b', version='master', conf_dir='', grid_system='FNAL', group='genie', +def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='', tune='G18_02_02_11b', gen_list='all',version='master', conf_dir='', grid_system='FNAL', group='genie', arch='SL6.x86_64', production='routine_validation', cycle='01', softw_topdir=os.getenv('GENIE_MASTER_DIR'), genie_topdir=os.getenv('GENIE'), grid_setup = os.getenv('GENIE')+'src/scripts/production/python/setup_FNAL.sh', genie_setup = os.getenv('GENIE')+'src/scripts/production/python/setup_GENIE.sh', jobs_topdir=os.getenv('PWD'), add_list=False, add_nucleons = False, time=2, git_branch="master", git_loc="https://github.com/GENIE-MC/Generator" ) : - + # Store root output only for vA spilnes: root_output = False if group_vN == False: root_output = True if not os.path.exists(xml_dir) : - print ( xml_dir+"doesn't exist") + print ( xml_dir+" doesn't exist") return + store_total_xsec = False + if gen_list == 'none' : + store_total_xsec = True + + + if group_vN == True : + process_name = "group_vN" + job_ID = 1 + else : + process_name = "group_vA" + job_ID = 3 + if mother_dir != '' : if os.path.exists(mother_dir) : xml_files_motherdir = glob.glob(mother_dir+"/*.xml") else : - print ( mother_dir+"doesn't exist") + print ( mother_dir+" doesn't exist") return #Given a mother directory and a daughter directory, the script tryies @@ -79,10 +91,18 @@ def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='' # Check if exist in xml_dir xml_file_name = os.path.basename(xml_file) if os.path.exists(xml_dir+"/"+xml_file_name[:-4]+".sh") : continue - if xml_file_name[:-4] == 'total_xsec' : continue - os.link(xml_file,xml_dir+"/"+xml_file_name) # link xml files - os.link(xml_file,xml_dir+"/"+xml_file_name[:-4]+".sh") # link sh files + if xml_file_name[:-4] == 'total_xsec' : + if store_total_xsec == True : + os.link(xml_file,xml_dir+"/"+xml_file_name) # link xml files + continue + os.link(xml_file,xml_dir+"/"+xml_file_name) # link xml files + os.link(xml_file[:-4]+".sh",xml_dir+"/"+xml_file_name[:-4]+".sh") # link sh files + if store_total_xsec == True : + temp_command_dict = {} + temp_command_dict[job_ID] = [] + return temp_command_dict + # Get names of sh files: these determine the name of the future xml files xml_files_dir = glob.glob(xml_dir+"/*.sh") @@ -181,8 +201,9 @@ def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='' out_files = [ "total_xsec.xml" ] if os.path.exists( xml_dir + '/total_xsec.xml' ) : - # Need to remove xml files before re-generating them - os.remove( xml_dir + '/total_xsec.xml' ) + if store_total_xsec == False : + # Need to remove xml files before re-generating them + os.remove( xml_dir + '/total_xsec.xml' ) if root_output : # Check if file exists - and remove @@ -206,13 +227,6 @@ def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='' commands.append( "gspl2root -p "+str_probe_list+" -t "+str_tgt_list+" -f "+path+"total_xsec.xml -o "+path+"total_xsec.root --tune "+tune ) out_files.append("total_xsec.root") - if group_vN == True : - process_name = "group_vN" - job_ID = 1 - else : - process_name = "group_vA" - job_ID = 3 - # Call Commands shell_file = '' command_list = [] @@ -223,5 +237,5 @@ def GroupSplineCommands( group_vN=False, xml_dir=os.getenv('PWD'), mother_dir='' ## Add command list to dictionary; command_dict = {} - command_dict[job_ID] = command_list ; - return command_dict ; + command_dict[job_ID] = command_list + return command_dict