diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index fc1b54defc6e7..ee7a21478aa01 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -33,7 +33,7 @@ class Options: defaultOptions.name = "NO NAME GIVEN" defaultOptions.evt_type = "" defaultOptions.filein = "" -defaultOptions.dbsquery="" +defaultOptions.dasquery="" defaultOptions.secondfilein = "" defaultOptions.customisation_file = [] defaultOptions.customise_commands = "" @@ -116,13 +116,13 @@ def filesFromList(fileName,s=None): print "found parent files:",sec return (prim,sec) -def filesFromDBSQuery(query,s=None): +def filesFromDASQuery(query,s=None): import os import FWCore.ParameterSet.Config as cms prim=[] sec=[] print "the query is",query - for line in os.popen('dbs search --query "%s"'%(query)): + for line in os.popen('das_client.py --query "%s"'%(query)): if line.count(".root")>=2: #two files solution... entries=line.replace("\n","").split() @@ -335,8 +335,8 @@ def filesFromOption(self): print "entry",entry if entry.startswith("filelist:"): filesFromList(entry[9:],self.process.source) - elif entry.startswith("dbs:"): - filesFromDBSQuery('find file where dataset = %s'%(entry[4:]),self.process.source) + elif entry.startswith("dbs:") or entry.startswith("das:"): + filesFromDASQuery('file dataset = %s'%(entry[4:]),self.process.source) else: self.process.source.fileNames.append(self._options.dirin+entry) if self._options.secondfilein: @@ -346,12 +346,12 @@ def filesFromOption(self): print "entry",entry if entry.startswith("filelist:"): self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0]) - elif entry.startswith("dbs:"): - self.process.source.secondaryFileNames.extend((filesFromDBSQuery('find file where dataset = %s'%(entry[4:])))[0]) + elif entry.startswith("dbs:") or entry.startswith("das:"): + self.process.source.secondaryFileNames.extend((filesFromDASQuery('file dataset = %s'%(entry[4:])))[0]) else: self.process.source.secondaryFileNames.append(self._options.dirin+entry) - if self._options.filein or self._options.dbsquery: + if self._options.filein or self._options.dasquery: if self._options.filetype == "EDM": self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(), @@ -387,9 +387,9 @@ def filesFromOption(self): if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"): self.process.source.processingMode = cms.untracked.string("RunsAndLumis") - if self._options.dbsquery!='': + if self._options.dasquery!='': self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring()) - filesFromDBSQuery(self._options.dbsquery,self.process.source) + filesFromDASQuery(self._options.dasquery,self.process.source) if self._options.inputCommands: if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() @@ -629,8 +629,8 @@ def addStandardSequences(self): mixingDict.pop('file') if self._options.pileup_input: - if self._options.pileup_input.startswith('dbs'): - mixingDict['F']=filesFromDBSQuery('find file where dataset = %s'%(self._options.pileup_input[4:],))[0] + if self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:'): + mixingDict['F']=filesFromDASQuery('file dataset = %s'%(self._options.pileup_input[4:],))[0] else: mixingDict['F']=self._options.pileup_input.split(',') specialization=defineMixing(mixingDict,self._options.fast) @@ -1252,7 +1252,7 @@ def prepare_GEN(self, sequence = None): except: loadFailure=True #if self.process.source and self.process.source.type_()=='EmptySource': - if not (self._options.filein or self._options.dbsquery): + if not (self._options.filein or self._options.dasquery): raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration") if not loadFailure: @@ -2087,7 +2087,7 @@ def prepare(self, doChecking = False): if hasattr(self.process.source,"secondaryFileNames"): if len(self.process.source.secondaryFileNames.value()): ioJson['secondary']=self.process.source.secondaryFileNames.value() - if self._options.pileup_input and self._options.pileup_input.startswith('dbs'): + if self._options.pileup_input and (self._options.pileup_input.startswith('dbs:') or self._options.pileup_input.startswith('das:')): ioJson['pileup']=self._options.pileup_input[4:] for (o,om) in self.process.outputModules_().items(): ioJson[o]=om.fileName.value() diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 83d38b2af1e95..47623233cefe7 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -251,10 +251,15 @@ default=defaultOptions.particleTable, dest="particleTable") +expertSettings.add_option("--dasquery", + help="Allow to define the source.fileNames from the das search command", + default='', + dest="dasquery") + expertSettings.add_option("--dbsquery", - help="Allow to define the source.fileNames from the dbs search command", + help="Deprecated. Please use dasquery option. Functions for backward compatibility", default='', - dest="dbsquery") + dest="dasquery") expertSettings.add_option("--lazy_download", help="Enable lazy downloading of input files", diff --git a/Configuration/PyReleaseValidation/python/MatrixReader.py b/Configuration/PyReleaseValidation/python/MatrixReader.py index bcdbd3e199b21..14b0fe814c591 100644 --- a/Configuration/PyReleaseValidation/python/MatrixReader.py +++ b/Configuration/PyReleaseValidation/python/MatrixReader.py @@ -346,7 +346,7 @@ def showRaw(self, useInput, refRel=None, fromScratch=None, what='all',step1Only= for (index,s) in enumerate(indexAndSteps): for (stepName,cmd) in s: stepIndex=index+1 - if 'dbsquery.log' in cmd: continue + if 'dasquery.log' in cmd: continue line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd line=line.replace('DQMROOT','DQM') outFile.write(line+'\n') diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index 882430b33308d..3b2b6ed4141a0 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -94,27 +94,27 @@ def closeCmd(i,ID): self.stat.append('NOTRUN') aborted=True continue - #create lumiRange file first so if dbs fails we get its error code + #create lumiRange file first so if das fails we get its error code cmd2 = com.lumiRanges() if cmd2: cmd2 =cmd+cmd2+closeCmd(istep,'lumiRanges') lumiRangeFile='step%d_lumiRanges.log'%(istep,) retStep = self.doCmd(cmd2) - cmd+=com.dbs() - cmd+=closeCmd(istep,'dbsquery') + cmd+=com.das() + cmd+=closeCmd(istep,'dasquery') retStep = self.doCmd(cmd) - #don't use the file list executed, but use the dbs command of cmsDriver for next step - inFile='filelist:step%d_dbsquery.log'%(istep,) + #don't use the file list executed, but use the das command of cmsDriver for next step + inFile='filelist:step%d_dasquery.log'%(istep,) print "---" else: #chaining IO , which should be done in WF object already and not using stepX.root but .root cmd += com if self.noRun: cmd +=' --no_exec' - if inFile: #in case previous step used DBS query (either filelist of dbs:) + if inFile: #in case previous step used DAS query (either filelist of das:) cmd += ' --filein '+inFile inFile=None - if lumiRangeFile: #DBS query can also restrict lumi range + if lumiRangeFile: #DAS query can also restrict lumi range cmd += ' --lumiToProcess '+lumiRangeFile lumiRangeFile=None if 'HARVESTING' in cmd and not '134' in str(self.wf.numId) and not '--filein' in cmd: diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 8a584c9ba8409..34ff3e40db4a9 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -50,7 +50,7 @@ def __init__(self,dataSet,label='',run=[],files=1000,events=InputInfoNDefault,sp self.ib_blacklist = ib_blacklist self.ib_block = ib_block - def dbs(self): + def das(self): query_by = "block" if self.ib_block else "dataset" query_source = "{0}#{1}".format(self.dataSet, self.ib_block) if self.ib_block else self.dataSet if len(self.run) is not 0: @@ -890,9 +890,9 @@ def genvalid(fragment,d,suffix='all',fi=''): steps['ZJetsLNu_Tune4C_8TeV_madgraph-pythia8']=genvalid('Hadronizer_MgmMatchTune4C_8TeV_madgraph_pythia8_cff',step1GenDefaults,fi=5591) steps['ReggeGribovPartonMC_EposLHC_5TeV_pPb']=genvalid('GeneratorInterface/ReggeGribovPartonMCInterface/ReggeGribovPartonMC_EposLHC_5TeV_pPb_cfi',step1GenDefaults) -PU={'-n':10,'--pileup':'default','--pileup_input':'dbs:/RelValMinBias/%s/GEN-SIM'%(baseDataSetRelease[0],)} -PU25={'-n':10,'--pileup':'AVE_10_BX_25ns_m8','--pileup_input':'dbs:/RelValMinBias_13/%s/GEN-SIM'%(baseDataSetRelease[7],)} -PU50={'-n':10,'--pileup':'AVE_20_BX_50ns_m8','--pileup_input':'dbs:/RelValMinBias_13/%s/GEN-SIM'%(baseDataSetRelease[7],)} +PU={'-n':10,'--pileup':'default','--pileup_input':'das:/RelValMinBias/%s/GEN-SIM'%(baseDataSetRelease[0],)} +PU25={'-n':10,'--pileup':'AVE_10_BX_25ns_m8','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(baseDataSetRelease[7],)} +PU50={'-n':10,'--pileup':'AVE_20_BX_50ns_m8','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(baseDataSetRelease[7],)} PUFS={'--pileup':'default'} PUFS2={'--pileup':'mix_2012_Startup_inTimeOnly'} steps['TTbarFSPU']=merge([PUFS,Kby(100,500),steps['TTbarFS']] ) @@ -1281,7 +1281,7 @@ def genvalid(fragment,d,suffix='all',fi=''): '--data':'', '--scenario':'pp', '--filein':'file:step2.root', - '--secondfilein':'filelist:step1_dbsquery.log'} + '--secondfilein':'filelist:step1_dasquery.log'} steps['SKIMDreHLT'] = merge([ {'--conditions':'auto:com10_%s'%menu,'--filein':'file:step3.root'}, steps['SKIMD'] ]) @@ -1290,7 +1290,7 @@ def genvalid(fragment,d,suffix='all',fi=''): '--data':'', '--scenario':'cosmics', '--filein':'file:step2.root', - '--secondfilein':'filelist:step1_dbsquery.log'} + '--secondfilein':'filelist:step1_dasquery.log'} #### for special wfs ### @@ -1322,7 +1322,7 @@ def genvalid(fragment,d,suffix='all',fi=''): '--process':'rereRECO', '--datatier':'AOD', '--eventcontent':'AOD', - '--secondfilein':'filelist:step1_dbsquery.log', + '--secondfilein':'filelist:step1_dasquery.log', }, steps['RECOD']])