diff --git a/Configuration/PyReleaseValidation/python/MatrixReader.py b/Configuration/PyReleaseValidation/python/MatrixReader.py index e4311cd543fe5..be5c546246275 100644 --- a/Configuration/PyReleaseValidation/python/MatrixReader.py +++ b/Configuration/PyReleaseValidation/python/MatrixReader.py @@ -28,6 +28,9 @@ def __init__(self, opt): self.noRun = opt.noRun self.checkInputs = opt.checkInputs + # -l restricts the expansion to the requested workflows; the listing, the + # runner and the wmagent injection all act on that same selection + self.selected = set(opt.testList) if opt.testList else None return def reset(self, what='all'): @@ -102,23 +105,24 @@ def reset(self, what='all'): def makeCmd(self, step): - cmd = '' + cmd = [] cfg = None input = None for k,v in step.items(): if 'no_exec' in k : continue # we want to really run it ... - if k.lower() == 'cfg': + klow = k.lower() + if klow == 'cfg': cfg = v continue # do not append to cmd, return separately - if k.lower() == 'input': - input = v + if klow == 'input': + input = v continue # do not append to cmd, return separately - + #chain the configs #if k.lower() == '--python': # v = 'step%d_%s'%(index,v) - cmd += ' ' + k + ' ' + str(v) - return cfg, input, cmd + cmd.append(' ' + k + ' ' + str(v)) + return cfg, input, ''.join(cmd) def makeStep(self,step,overrides): from Configuration.PyReleaseValidation.relval_steps import merge @@ -143,8 +147,8 @@ def verifyDefaultInputs(self): ============================================================================= """.format(sys._getframe(1).f_lineno - 1,wf[0],wf)) - def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None): - + def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None, selected=None): + prefix = self.filesPrefMap[fileNameIn] print("processing", fileNameIn) @@ -204,7 +208,12 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None): [(x,refRel) for x in self.relvalModule.baseDataSetRelease] ) + useIBEos = os.getenv("CMSSW_USE_IBEOS","false")=="true" + madeCmds = {} + for num, wfInfo in self.relvalModule.workflows.items(): + if selected is not None and num not in selected: + continue commands=[] wfName = wfInfo[0] stepList = wfInfo[1] @@ -295,8 +304,16 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None): from Configuration.PyReleaseValidation.relval_steps import merge copyStep=merge(addCom+[self.makeStep(self.relvalModule.steps[stepName],stepOverrides)]) cfg, input, opts = self.makeCmd(copyStep) - else: + elif stepOverrides: cfg, input, opts = self.makeCmd(self.makeStep(self.relvalModule.steps[stepName],stepOverrides)) + else: + # without overrides a step yields the same command in every + # workflow that runs it + made = madeCmds.get(stepName) + if made is None: + made = self.makeCmd(self.relvalModule.steps[stepName]) + madeCmds[stepName] = made + cfg, input, opts = made if input and cfg : msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName @@ -322,7 +339,7 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None): if self.wm and self.revertDqmio=='yes': cmd=cmd.replace('DQMIO','DQM') cmd=cmd.replace('--filetype DQM','') - if os.getenv("CMSSW_USE_IBEOS","false")=="true": + if useIBEos: cmd="export CMSSW_USE_IBEOS=true; "+cmd commands.append(cmd) ranStepList.append(stepName) @@ -526,13 +543,13 @@ def prepare(self, useInput=None, refRel='', fromScratch=None): continue try: - self.readMatrix(matrixFile, useInput, refRel, fromScratch) + self.readMatrix(matrixFile, useInput, refRel, fromScratch, self.selected) if self.checkInputs: self.verifyDefaultInputs() except Exception as e: print("ERROR reading file:", matrixFile, str(e)) raise - + try: self.createWorkFlows(matrixFile) except Exception as e: diff --git a/Configuration/PyReleaseValidation/python/MatrixUtil.py b/Configuration/PyReleaseValidation/python/MatrixUtil.py index cab4dc7ed30dc..62f41395fd3ed 100644 --- a/Configuration/PyReleaseValidation/python/MatrixUtil.py +++ b/Configuration/PyReleaseValidation/python/MatrixUtil.py @@ -1,3 +1,4 @@ +import copy import os import subprocess @@ -7,13 +8,106 @@ def __setitem__(self,key,value): print("ERROR in Matrix") print("overwriting",key,"not allowed") else: - self.update({float(key):WF(float(key),value)}) + num=float(key) + dict.__setitem__(self,num,WF(num,value)) def addOverride(self,key,override): self[key].addOverride(override) +# a fragment step, held as (cfg, howMuch, base) and turned into its dictionary the +# first time it is read; the upgrade matrix holds a couple of million of them and a +# run reads a few percent +class DeferredFragmentStep(tuple): + __slots__ = () + # same result, same key order, as merge([{'cfg':cfg},howMuch,base]) + def expand(self): + cfg,howMuch,base = self + step = copy.copy(base) + step.update(howMuch) + step['cfg'] = cfg + return step + #the class to collect all possible steps +# the upgrade matrix holds some two million fragment steps grouped in about 180 +# families, one family per step name spanning every fragment and every upgrade key. +# every name a family defines ends in '_'+family, so a name identifies its own family, +# and a family is built the first time one of its names is asked for. anything that +# walks the whole dictionary builds every family first. class Steps(dict): + def __init__(self,*args,**kwargs): + dict.__init__(self,*args,**kwargs) + self.familyBuilder = None + self.pendingFamilies = {} + self.familiesByTail = {} + self.buildingFamily = False + + def deferFamily(self,family,rows): + self.pendingFamilies[family] = rows + self.familiesByTail.setdefault(family.rsplit('_',1)[-1],[]).append(family) + + def buildFamily(self,rows): + self.buildingFamily = True + try: + for row in rows: + self.familyBuilder(*row) + finally: + self.buildingFamily = False + + def buildFamiliesFor(self,name): + # build every pending family that could define name; True when one was built + if not self.pendingFamilies: + return False + if name.endswith('INPUT'): + name = name[:-len('INPUT')] + built = False + for family in self.familiesByTail.get(name.rsplit('_',1)[-1],()): + if not name.endswith('_'+family): + continue + rows = self.pendingFamilies.pop(family,None) + if rows is not None: + self.buildFamily(rows) + built = True + return built + + def buildAllFamilies(self): + while self.pendingFamilies: + family = next(iter(self.pendingFamilies)) + self.buildFamily(self.pendingFamilies.pop(family)) + + def __missing__(self,key): + if self.buildFamiliesFor(key): + return dict.__getitem__(self,key) + raise KeyError(key) + + def __contains__(self,key): + if dict.__contains__(self,key): + return True + # while a family is being built its own names are new, and the only other + # membership test made there is for a name that belongs to no family + if self.buildingFamily: + return False + return self.buildFamiliesFor(key) and dict.__contains__(self,key) + + def __iter__(self): + self.buildAllFamilies() + return dict.__iter__(self) + + def __len__(self): + self.buildAllFamilies() + return dict.__len__(self) + + def keys(self): + self.buildAllFamilies() + return dict.keys(self) + + def values(self): + self.buildAllFamilies() + return dict.values(self) + + def items(self): + self.buildAllFamilies() + return dict.items(self) + def __setitem__(self,key,value): if key in self: print("ERROR in Step") @@ -21,10 +115,24 @@ def __setitem__(self,key,value): import sys sys.exit(-9) else: - self.update({key:value}) + dict.__setitem__(self,key,value) # make the python file named .py #if not '--python' in value: self[key].update({'--python':'%s.py'%(key,)}) + def __getitem__(self,key): + value=dict.__getitem__(self,key) + if type(value) is DeferredFragmentStep: + value=value.expand() + dict.__setitem__(self,key,value) + return value + + # dict.get would bypass __getitem__ and hand out an unexpanded step + def get(self,key,default=None): + try: + return self[key] + except KeyError: + return default + def overwrite(self,keypair): value=self[keypair[1]] self.update({keypair[0]:value}) @@ -210,23 +318,14 @@ def __str__(self): # merge dictionaries, with priority on the [0] index +# the result keeps the type and the key order of the last item, extended with the +# keys each earlier item adds, and the value of the earliest item that defines a key def merge(dictlist,TELL=False): - import copy - last=len(dictlist)-1 - if TELL: print(last,dictlist) - if last==0: - # ONLY ONE ITEM LEFT - return copy.copy(dictlist[0]) - else: - reducedlist=dictlist[0:max(0,last-1)] - if TELL: print(reducedlist) - # make a copy of the last item - d=copy.copy(dictlist[last]) - # update with the last but one item - d.update(dictlist[last-1]) - # and recursively do the rest - reducedlist.append(d) - return merge(reducedlist,TELL) + if TELL: print(len(dictlist)-1,dictlist) + d=copy.copy(dictlist[-1]) + for i in range(len(dictlist)-2,-1,-1): + d.update(dictlist[i]) + return d def remove(d,key,TELL=False): import copy diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 3eaa1d4f4d364..4dfdc7ea002e6 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -2,7 +2,7 @@ from copy import deepcopy from functools import partial -from .MatrixUtil import Steps, merge, remove, Kby, Mby, genvalid, InputInfo, selectedLS, stCond +from .MatrixUtil import Steps, DeferredFragmentStep, merge, remove, Kby, Mby, genvalid, InputInfo, selectedLS, stCond from Configuration.HLT.autoHLT import autoHLT from Configuration.AlCa.autoPCL import autoPCL @@ -5046,55 +5046,68 @@ def gen2024HiMix(fragment,howMuch): # in case special WF has PU-specific changes: apply *after* basic PU step is created specialWF.setupPU(upgradeStepDict, k, upgradeProperties[year][k]) - -for step in upgradeStepDict.keys(): + +allUpgradeKeys = [key for year in upgradeKeys for key in upgradeKeys[year]] +# whether a key can recycle a GEN-SIM input depends on the key alone +recyclingKeys = [key for key in allUpgradeKeys if "Run4"+defaultRun4Geometry in key and 'FS' not in key and defaultDataSets[key] != ''] + +# all the steps one fragment contributes to one step name; every name it defines ends in +# '_'+step, which is what lets Steps build a step name's family on demand +def makeFragmentSteps(step,stepDict,isPremix,isHybridPU,istepDict,frag,info): + fragName=frag[:-4] + howMuch=info.howMuch + for key in allUpgradeKeys: + k=fragName+'_'+key+'_'+step + if key in stepDict: + if stepDict[key] is None: + steps[k]=None + elif isPremix: + # Include premixing stage1 only for SingleNu, use special step name + if not 'SingleNu' in frag: + continue + stepKey = 'PREMIX_'+key+'_'+step + howMuch = Kby(100,100) + steps[stepKey]=merge([ {'--evt_type':frag},howMuch,stepDict[key]]) + else: + steps[k]=DeferredFragmentStep((frag,howMuch,stepDict[key])) + #get inputs in case of -i...but no need to specify in great detail + #however, there can be a conflict of beam spots but this is lost in the dataset name + #so please be careful + # pre-Run4 input recycling is DISABLED + if key in recyclingKeys and 'FastSim' not in k: + s=fragName+'_'+key + if s+'INPUT' not in steps and s in baseDataSetReleaseBetter and \ + (istepDict is None or key not in istepDict or istepDict[key] is not None): + steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN-SIM'%(baseDataSetReleaseBetter[s],),location='STD')} + # begin COMMENT: reads old format file + # else: #For FastSim to recycle GEN + # steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN'%(baseDataSetReleaseBetter[s],),location='STD')} + # end COMMENT: reads old format file + # this condition is checked here to avoid skipping the creation of default steps for other fragments + if isHybridPU: + # minbias fastsim for PU mixing + if not 'MinBias_14TeV' in frag: + continue + stepKey = 'HYBRID_'+key+'_'+step + howMuch = Kby(100,100) + steps[stepKey]=merge([ {'--evt_type':frag},howMuch,stepDict[key]]) + +steps.familyBuilder = makeFragmentSteps + +for step,stepDict in upgradeStepDict.items(): # we need to do this for each fragment if ('Sim' in step and ('Fast' not in step and step != 'Sim')) or ('Premix' in step) or ('Sim' not in step and 'Gen' in step): - for frag,info in upgradeFragments.items(): - howMuch=info.howMuch - for key in [key for year in upgradeKeys for key in upgradeKeys[year]]: - k=frag[:-4]+'_'+key+'_'+step - if step in upgradeStepDict and key in upgradeStepDict[step]: - if upgradeStepDict[step][key] is None: - steps[k]=None - elif 'Premix' in step: - # Include premixing stage1 only for SingleNu, use special step name - if not 'SingleNu' in frag: - continue - stepKey = 'PREMIX_'+key+'_'+step - howMuch = Kby(100,100) - steps[stepKey]=merge([ {'--evt_type':frag},howMuch,upgradeStepDict[step][key]]) - else: - steps[k]=merge([ {'cfg':frag},howMuch,upgradeStepDict[step][key]]) - #get inputs in case of -i...but no need to specify in great detail - #however, there can be a conflict of beam spots but this is lost in the dataset name - #so please be careful - s=frag[:-4]+'_'+key - # exclude upgradeKeys without input dataset, and special WFs that disable reuse - istep = step+preventReuseKeyword - - if 'FastSim' not in k and s+'INPUT' not in steps and s in baseDataSetReleaseBetter and defaultDataSets[key] != '' and \ - (istep not in upgradeStepDict or key not in upgradeStepDict[istep] or upgradeStepDict[istep][key] is not None) and "Run4"+defaultRun4Geometry in key: - # pre-Run4 input recycling is DISABLED - if 'FS' not in key: #For FullSim - steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN-SIM'%(baseDataSetReleaseBetter[s],),location='STD')} - # begin COMMENT: reads old format file - # else: #For FastSim to recycle GEN - # steps[k+'INPUT']={'INPUT':InputInfo(dataSet='/RelVal'+info.dataset+'/%s/GEN'%(baseDataSetReleaseBetter[s],),location='STD')} - # end COMMENT: reads old format file - # this condition is checked here to avoid skipping the creation of default steps for other fragments - if 'HybridPU' in step: - # minbias fastsim for PU mixing - if not 'MinBias_14TeV' in frag: - continue - stepKey = 'HYBRID_'+key+'_'+step - howMuch = Kby(100,100) - steps[stepKey]=merge([ {'--evt_type':frag},howMuch,upgradeStepDict[step][key]]) + # exclude special WFs that disable reuse + istepDict = upgradeStepDict.get(step+preventReuseKeyword) + isPremix = 'Premix' in step + isHybridPU = 'HybridPU' in step + steps.deferFamily(step,[(step,stepDict,isPremix,isHybridPU,istepDict,frag,info) + for frag,info in upgradeFragments.items()]) else: - for key in [key for year in upgradeKeys for key in upgradeKeys[year]]: - k=step+'_'+key - if step in upgradeStepDict and key in upgradeStepDict[step]: - if upgradeStepDict[step][key] is None: + for key in allUpgradeKeys: + if key in stepDict: + k=step+'_'+key + if stepDict[key] is None: steps[k]=None else: - steps[k]=merge([upgradeStepDict[step][key]]) + steps[k]=merge([stepDict[key]]) diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 3eee14a385208..6d73f36b8d145 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -21,57 +21,107 @@ def makeStepName(key,frag,step,suffix): def notForGenOnly(key,specialType): return "GenOnly" in key and specialType != 'baseline' +# the special workflows that customise a given step; this depends on the step name only, +# and the step names come from a small fixed set, so answer it once per step name +customisersByStep = {} +def customisersFor(step): + customisers = customisersByStep.get(step) + if customisers is None: + isPU = 'PU' in step + stepNoPU = step.replace('PU','') if isPU else None + customisers = [(specialType,specialWF) for specialType,specialWF in upgradeWFs.items() + if specialType != 'baseline' and ((isPU and stepNoPU in specialWF.PU) or (step in specialWF.steps))] + customisersByStep[step] = customisers + return customisers + for year in upgradeKeys: for i,key in enumerate(upgradeKeys[year]): numWF=numWFAll[year][i] + # neither the applicable flavors nor the presence of a harvesting step depend on + # the fragment, so resolve them once per key + activeWFs = [(specialType,specialWF) for specialType,specialWF in upgradeWFs.items() + if not notForGenOnly(key,specialType)] + scenToRun = upgradeProperties[year][key]['ScenToRun'] + hasHarvest = any('HARVEST' in step for step in scenToRun) + # the resolved step names and the steps each flavor customises depend on the + # fragment only through the flags below, so resolve them once per combination + byFragmentClass={} for frag,info in upgradeFragments.items(): # phase2-specific fragments are skipped in phase1 if ("CE_E" in frag or "CE_H" in frag) and year==2017: numWF += 1 continue - stepList={} - for specialType in upgradeWFs.keys(): - if notForGenOnly(key,specialType): - continue - stepList[specialType] = [] - hasHarvest = False - for step in upgradeProperties[year][key]['ScenToRun']: - stepMaker = makeStepName - if 'Sim' in step and 'Fast' not in step and step != "Sim": - if 'DisplacedParticleGun' in frag: - step = 'GenSimDisplaced' - elif 'HLBeamSpot' in step: - if '14TeV' in frag: - step = 'GenSimHLBeamSpot14' - elif 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag: - step = 'GenSimHLBeamSpotCloseBy' - elif 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag: - step = 'GenSimCloseBy' - stepMaker = makeStepNameSim - elif 'Gen' in step: - if 'HLBeamSpot' in step: - if '14TeV' in frag: - step = 'GenHLBeamSpot14' - stepMaker = makeStepNameSim + fragName = frag[:-4] + is14TeV = '14TeV' in frag + isCloseBy = 'CloseBy' in frag or 'CE_E' in frag or 'CE_H' in frag + isDisplaced = 'DisplacedParticleGun' in frag + if (is14TeV,isCloseBy,isDisplaced) in byFragmentClass: + resolved,customised = byFragmentClass[(is14TeV,isCloseBy,isDisplaced)] + else: + resolved=[] + for step in scenToRun: + stepMaker = makeStepName + if 'Sim' in step and 'Fast' not in step and step != "Sim": + if isDisplaced: + step = 'GenSimDisplaced' + elif 'HLBeamSpot' in step: + if is14TeV: + step = 'GenSimHLBeamSpot14' + elif isCloseBy: + step = 'GenSimHLBeamSpotCloseBy' + elif isCloseBy: + step = 'GenSimCloseBy' + stepMaker = makeStepNameSim + elif 'Gen' in step: + if 'HLBeamSpot' in step: + if is14TeV: + step = 'GenHLBeamSpot14' + stepMaker = makeStepNameSim + resolved.append((stepMaker,step)) + # the steps each flavor customises: a flavor that customises none of them + # ends up with the baseline step list, which workflow_() drops as spurious + customised={} + for index,(stepMaker,step) in enumerate(resolved): + for specialType,specialWF in customisersFor(step): + customised.setdefault(specialType,[]).append(index) + byFragmentClass[(is14TeV,isCloseBy,isDisplaced)] = (resolved,customised) - if 'HARVEST' in step: hasHarvest = True - for specialType,specialWF in upgradeWFs.items(): - if notForGenOnly(key,specialType): ## we don't need all the flavors for the GEN - continue + baseStepList = [stepMaker(key,fragName,step,'') for stepMaker,step in resolved] - if (specialType != 'baseline') and ( ('PU' in step and step.replace('PU','') in specialWF.PU) or (step in specialWF.steps) ): - stepList[specialType].append(stepMaker(key,frag[:-4],step,specialWF.suffix)) + for specialType,specialWF in activeWFs: + accepted = None + if specialType=='baseline': + stepList = list(baseStepList) + else: + modified = customised.get(specialType) + # PMXS1 truncates its list below, so it differs from the baseline one + # even when it customises no step + if modified is None and specialType!="PMXS1": + continue + # a flavor whose condition() rejects this workflow contributes + # nothing, so its step list is never needed + if not specialWF.conditionUsesStepList: + accepted = specialWF.condition(info.dataset, None, key, hasHarvest) + if not accepted: + continue + if modified is None: modified = [] + stepList = [] + for index,(stepMaker,step) in enumerate(resolved): + if index not in modified: + stepList.append(baseStepList[index]) + continue + stepList.append(stepMaker(key,fragName,step,specialWF.suffix)) # hack to add an extra step if 'ProdLike' in specialType: if 'Reco' in step: # handles both Reco, RecoFakeHLT and RecoGlobal stepWoFakeHLT = step.replace('FakeHLT','') # ignore "FakeHLT" from step - stepList[specialType].append(stepMaker(key,frag[:-4],stepWoFakeHLT.replace('RecoGlobal','MiniAOD').replace('RecoNano','MiniAOD').replace('Reco','MiniAOD'),specialWF.suffix)) + stepList.append(stepMaker(key,fragName,stepWoFakeHLT.replace('RecoGlobal','MiniAOD').replace('RecoNano','MiniAOD').replace('Reco','MiniAOD'),specialWF.suffix)) if 'RecoNano' in stepWoFakeHLT: - stepList[specialType].append(stepMaker(key,frag[:-4],stepWoFakeHLT.replace('RecoNano','Nano'),specialWF.suffix)) + stepList.append(stepMaker(key,fragName,stepWoFakeHLT.replace('RecoNano','Nano'),specialWF.suffix)) # hack to add extra HLT75e33 step for Phase-2 if 'HLT75e33' in specialType: if 'RecoGlobal' in step: - stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','HLT75e33'),specialWF.suffix)) + stepList.append(stepMaker(key,fragName,step.replace('RecoGlobal','HLT75e33'),specialWF.suffix)) # similar hacks for premixing if 'PMX' in specialType: if 'GenSim' in step or 'Gen' in step: @@ -79,9 +129,9 @@ def notForGenOnly(key,specialType): if step in specialWF.PU: stepMade = stepMaker(key,'PREMIX',s,specialWF.suffix) # append for combined - if 'S2' in specialType: stepList[specialType].append(stepMade) + if 'S2' in specialType: stepList.append(stepMade) # replace for s1 - else: stepList[specialType][-1] = stepMade + else: stepList[-1] = stepMade # similar hack for fastpu if 'HybridPU' in specialType: if 'GenSim' in step: @@ -89,14 +139,12 @@ def notForGenOnly(key,specialType): if step in specialWF.PU: stepMade = stepMaker(key,'HYBRID',s,specialWF.suffix) # append for combined - if 'S2' in specialType: stepList[specialType].append(stepMade) - else: - stepList[specialType].append(stepMaker(key,frag[:-4],step,'')) - for specialType,specialWF in upgradeWFs.items(): - # remove other steps for premixS1 - if notForGenOnly(key,specialType): - continue - if specialType=="PMXS1": - stepList[specialType] = stepList[specialType][:1] - specialWF.workflow(workflows, numWF, info.dataset, stepList[specialType], key, hasHarvest) + if 'S2' in specialType: stepList.append(stepMade) + # remove other steps for premixS1 + if specialType=="PMXS1": + stepList = stepList[:1] + if accepted: + specialWF.workflow_(workflows, numWF, info.dataset, stepList, key) + else: + specialWF.workflow(workflows, numWF, info.dataset, stepList, key, hasHarvest) numWF+=1 diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 24c49dea8caa4..a84f1daf6de56 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -196,9 +196,13 @@ def workflow_(self, workflows, num, fragment, stepList, key): if self.offset==0 or workflows[num][1]!=stepList: workflows[num+self.offset] = [ fragmentTmp, stepList ] + # True when condition() inspects or amends stepList, so that the caller has to + # build the step list before asking + conditionUsesStepList = False + def condition(self, fragment, stepList, key, hasHarvest): return False - + def preventReuse(self, stepName, stepDict, k): if "Sim" in stepName and stepName != "Sim": stepDict[stepName][k] = None @@ -304,6 +308,7 @@ def setup_(self, step, stepName, stepDict, k, properties): stepDict[stepName][k] = merge([{'--filein': 'file:step3.root', '--secondfilein': 'file:step2.root'}, stepDict[step][k]]) if 'Digi' in step and 'NoHLT' not in step: stepDict[stepName][k] = merge([{'-s': re.sub(',HLT.*', '', stepDict[step][k]['-s'])}, stepDict[step][k]]) + conditionUsesStepList = True def condition(self, fragment, stepList, key, hasHarvest): if ('TTbar_14TeV' in fragment and '2022' == key): stepList.insert(stepList.index('Digi_DigiNoHLT_2022')+1, 'HLTRun3_2022') @@ -1287,15 +1292,16 @@ def __init__(self, digi = {}, reco = {}, mini = {}, harvest = {}, **kwargs): self.__mini = mini self.__harvest = harvest + years = run3_years + ['Run4'] + fragments = ["TTbar_14","ZMM_14","ZEE_14","ZTT_14","NuGun","SingleMu","QCD_Pt15To7000_Flat"] + def condition(self, fragment, stepList, key, hasHarvest): # select only a subset of the workflows - years = run3_years + ['Run4'] - fragments = ["TTbar_14","ZMM_14","ZEE_14","ZTT_14","NuGun","SingleMu","QCD_Pt15To7000_Flat"] - selected = [ - (any(y in key for y in years) and ('FS' not in key) and any( f in fragment for f in fragments)), + selected = ( + (any(y in key for y in self.years) and ('FS' not in key) and any( f in fragment for f in self.fragments)) or (('HI' in key) and ('Hydjet' in fragment) and ("PixelOnly" in self.suffix) ) - ] - result = any(selected) and hasHarvest + ) + result = selected and hasHarvest return result