Public Member Functions
	Init (self, options)

	count_python_processes (self, macroName)

	list_of_runs (self, macroName)

	getPartitions (self, runList, path)

	convert (self, runList, path, partitions={})

	getRunNrFromOffline (self, rMin=-1, rMax=99999)

	runDataQuality (self, latest)

	RerunDataQuality (self, runNrs=[], rMin=-1, rMax=99999)

	check4NewFiles (self, latest, rmax)

	runSinglePartition (self, path, r, p, EOScopy=False, check=True)

	checkFile (self, path, r, p)

	copy2EOS (self, path, success, overwrite)

	check (self, path, partitions)

	getFileList (self, p, latest, rmax=999999, minSize=10E6)

	getNumberOfEvents (self, r)

	checkEOS (self, copy=False, latest=4361, rlast=999999)

	getConvStats (self, runList)

	rawStats (self, runList)

	makeHistos (self, runList)

	mips (self)

Public Attributes
	Mext

	options

	runNrs

	dqDataFiles

	RawrunNrs

	eosInventory

	daqInventory

	missing

Static Public Attributes
str	command = "$SNDSW_ROOT/shipLHC/scripts/Survey-MufiScifi.py -r "+str(run)+" -p "+pathConv+" -g geofile_sndlhc_H6.root -c Mufi_Efficiency -n -1 -t DS"

Detailed Description

Definition at line 29 of file runProd.py.

Member Function Documentation

◆ check()

runProd.prodManager.check	(	self,
		path,
		partitions
	)

Definition at line 239 of file runProd.py.

   def check(self,path,partitions):
     success = {}
     for r in partitions:
        success[r] = []
        runNr   = str(r).zfill(6)
        for x in partitions[r]:
           p = str(x).zfill(4)
           rc = checkFile(path,runNr,p)
           if rc==0: success[r].append(x)
     return success
 

◆ check4NewFiles()

runProd.prodManager.check4NewFiles	(	self,
		latest,
		rmax
	)

Definition at line 157 of file runProd.py.

   def check4NewFiles(self,latest,rmax):
      rawDataFiles = self.getFileList(path,latest,rmax,minSize=10E6)
      convDataFiles = self.getFileList(pathConv,latest,rmax,minSize=0)
      orderedRDF = list(rawDataFiles.keys())
      orderedCDF = list(convDataFiles.keys())
      orderedRDF.sort(),orderedCDF.sort()
 
      for x in orderedRDF: 
           if x in orderedCDF: continue
           lpruns = self.list_of_runs('convertRawData')[1]
           if x in lpruns: continue
           r = x//10000 
           p = x%10000
           if r==4541 and p==38: continue   # corrupted raw data file
           print('executing run %i and partition %i'%(r,p))
           self.runSinglePartition(path,str(r).zfill(6),str(p).zfill(4),EOScopy=True,check=True)
 

◆ checkEOS()

runProd.prodManager.checkEOS	(	self,
		copy = `False`,
		latest = `4361`,
		rlast = `999999`
	)

Definition at line 308 of file runProd.py.

   def checkEOS(self,copy=False,latest=4361,rlast=999999):
       self.eosInventory = self.getFileList(path,latest,rlast)
       tmp = self.options.server 
       self.options.server = "root://snd-server-1.cern.ch/"
       self.daqInventory = self.getFileList('/mnt/raid5/data_online/',latest,rlast)
       self.options.server = tmp
       self.missing = {}
       for r in self.daqInventory:
            if not r in self.eosInventory:
               p = r%10000
               run = r//10000
               if not run in self.missing: self.missing[run]=[]
               self.missing[run].append(p)
       if copy:
          for r in self.missing:
               dirname ='run_'+str(r).zfill(6)
               for p in self.missing[r]:
                   filename = 'data_'+str(p).zfill(4)+'.root'
                   source = '/mnt/raid5/data_online/'+dirname+'/'+filename
                   target = path+dirname+'/'+filename
                   os.system("xrdcp -f "+source+" "+os.environ['EOSSHIP']+target)
       

◆ checkFile()

runProd.prodManager.checkFile	(	self,
		path,
		r,
		p
	)

Definition at line 208 of file runProd.py.

   def checkFile(self,path,r,p):
      print('checkfile',path,r,p)
      inFile = self.options.server+path+'run_'+ r+'/data_'+p+'.root'
      fI = ROOT.TFile.Open(inFile)
      if fI.Get('event'): Nraw = fI.Get("event").GetEntries()
      else: Nraw = fI.Get("data").GetEntries()
      outFile = 'sndsw_raw_'+r+'-'+p+self.Mext+'.root'
      fC = ROOT.TFile(outFile)
      test = fC.Get('rawConv')
      if not test:
          print('Error:',path,r,p,' rawConv not found')
          return -2       
      Nconv = fC.Get("rawConv").GetEntries()
      if Nraw != Nconv: 
          print('Error:',path,r,p,':',Nraw,Nconv)
          return -1
      return 0
 

◆ convert()

runProd.prodManager.convert	(	self,
		runList,
		path,
		partitions = `{}`
	)

Definition at line 76 of file runProd.py.

   def convert(self,runList,path,partitions={}):
      if len(partitions)==0: partitions = self.getPartitions(runList,path)
      for r in partitions:
         runNr   = str(r).zfill(6)
    # find partitions
         for p in partitions[r]:
             self.runSinglePartition(path,runNr,str(p).zfill(4),EOScopy=True,check=True)
 

◆ copy2EOS()

runProd.prodManager.copy2EOS	(	self,
		path,
		success,
		overwrite
	)

Definition at line 226 of file runProd.py.

   def copy2EOS(self,path,success,overwrite):
     for i in success:
       r = str(i).zfill(6)
       for x in success[i]:
         p = str(x).zfill(4)
         outFile = 'sndsw_raw_'+r+'-'+p+self.Mext+'.root'
         tmpPath = os.environ['EOSSHIP']+pathConv+"run_"+r+"/sndsw_raw-"+p+".root"
         print('copy '+outFile+'  '+tmpPath)
         command = 'xrdcp '
         if overwrite: command+=' -f '
         os.system(command+outFile+'  '+tmpPath)
         os.system('rm '+outFile)
 

◆ count_python_processes()

runProd.prodManager.count_python_processes	(	self,
		macroName
	)

Definition at line 37 of file runProd.py.

   def count_python_processes(self,macroName):
      username = pwd.getpwuid(os.getuid()).pw_name
      callstring = "ps -f -u " + username
# only works if screen is wide enough to print full name!
      status = subprocess.check_output(callstring,shell=True)
      n=0
      for x in status.decode().split('\n'):
         if not x.find(macroName)<0 and not x.find('python') <0: n+=1
      return n

◆ getConvStats()

runProd.prodManager.getConvStats	(	self,
		runList
	)

Definition at line 330 of file runProd.py.

   def getConvStats(self,runList):
     for run in runList:
       try: 
          f=ROOT.TFile.Open("sndsw_raw_"+str(run).zfill(6)+'.root')
          print(run,':',f.Get("rawConv").GetEntries())
       except:
          print('problem:',run)
         

◆ getFileList()

runProd.prodManager.getFileList	(	self,
		p,
		latest,
		rmax = `999999`,
		minSize = `10E6`
	)

Definition at line 250 of file runProd.py.

   def getFileList(self,p,latest,rmax=999999,minSize=10E6):
      inventory = {}
      dirList = str( subprocess.check_output("xrdfs "+self.options.server+" ls "+p,shell=True) )
      for x in dirList.split('\\n'):
          aDir = x[x.rfind('/')+1:]
          if not aDir.find('run')==0 or aDir.find('json')>0:continue
          runNr = int(aDir.split('_')[1])
          if not runNr > latest: continue
          if runNr > rmax:       continue
          fileList = str( subprocess.check_output("xrdfs "+self.options.server+" ls -l "+p+"/"+aDir,shell=True) )
          for z in fileList.split('\\n'):
               jj=0
               tmp = []
               for o in z.split(' '):
                  if not o=='': tmp.append(o)
               if self.options.server.find('snd-server')>0:
                  jj = 0
                  k = z.rfind('data_')
                  if not k>0: continue
                  if not z[k+9:k+10]=='.': continue
                  first = False
                  for x in z.split(' '):
                     if x=='': continue
                     if not first and x=='sndecs': first = True
                     if first and not x=='sndecs': 
                       size = int(x)
                       break
               else:
                  k = max(z.find('data_'),z.find('sndsw'))
                  if not k>0: continue
                  size = int(tmp[3+jj])
               if size<minSize: continue
               theDay = tmp[4+jj] 
               theTime = tmp[5+jj]
               fname = z[k:]
               run = int(aDir.split('_')[1])
               if run>900000: continue     # not a physical run
               k = fname.find('.root')
               partition = int(fname[k-4:k])
               d = theDay.split('-')
               t = theTime.split(':')
               gmt = time.mktime( (int(d[0]), int(d[1]), int(d[2]),  int(t[0]), int(t[1]), int(t[2]), 0, 0, 0 ) )
               inventory[run*10000+partition] = [aDir+"/"+fname,gmt]
      return inventory
 

◆ getNumberOfEvents()

runProd.prodManager.getNumberOfEvents	(	self,
		r
	)

Definition at line 295 of file runProd.py.

   def getNumberOfEvents(self,r):
# check for partitions
           dirlist  = str( subprocess.check_output("xrdfs "+options.server+" ls "+pathConv+"run_"+str(r).zfill(6),shell=True) )
           partitions = []
           for x in dirlist.split('\\n'):
                ix = x.find('sndsw_raw-')
                if ix<0: continue
                partitions.append(x[ix:])
           eventChain = ROOT.TChain('rawConv')
           for p in partitions:
                eventChain.Add(options.server+pathConv+'run_'+str(r).zfill(6)+'/'+p)
           return eventChain.GetEntries()
 

◆ getPartitions()

runProd.prodManager.getPartitions	(	self,
		runList,
		path
	)

Definition at line 63 of file runProd.py.

   def getPartitions(self,runList,path):
      partitions = {}
      for r in runList:
          directory = path+"run_"+str(r).zfill(6)
          dirlist  = str( subprocess.check_output("xrdfs "+options.server+" ls "+directory,shell=True) )
          partitions[r] = []
          for x in dirlist.split('\\n'):
             if x.find('.root')<0: continue
             ix = x.rfind('data_')
             if ix<0: continue
             partitions[r].append( int(x[ix+5:ix+9]) )
      return partitions
 

◆ getRunNrFromOffline()

runProd.prodManager.getRunNrFromOffline	(	self,
		rMin = `-1`,
		rMax = `99999`
	)

Definition at line 84 of file runProd.py.

   def getRunNrFromOffline(self,rMin=-1,rMax=99999):
      self.dqDataFiles = []
      hList = str( subprocess.check_output("xrdfs "+os.environ['EOSSHIP']+" ls /eos/experiment/sndlhc/www/offline",shell=True) )
      for x in hList.split('\\n'):
          if x.find('root')<0: continue
          run = x.split('/')[-1]
          if run.find('run')!=0: continue
          r = int(run[3:9])
          if r>=rMin and r<=rMax:  self.dqDataFiles.append(r)

◆ Init()

runProd.prodManager.Init	(	self,
		options
	)

Definition at line 31 of file runProd.py.

   def Init(self,options):
     self.Mext = ''  
     if options.FairTask_convRaw: 
        self.Mext = '_CPP'  
     self.options = options
     self.runNrs = []

◆ list_of_runs()

runProd.prodManager.list_of_runs	(	self,
		macroName
	)

Definition at line 46 of file runProd.py.

   def list_of_runs(self,macroName):
      lpruns = [[],[]]
      username = pwd.getpwuid(os.getuid()).pw_name
      callstring = "ps -f -u " + username
      status = subprocess.check_output(callstring,shell=True)
      for x in status.decode().split('\n'):
         if not x.find(macroName)<0 and not x.find('python') <0: 
            i = x.find('-r')
            k = x.find('-P')
            if i > 0:  
               r = int(x[i+3:].split(' ')[0])
               lpruns[0].append(r)
               if k>0: 
                  p = int(x[k+3:].split(' ')[0])
                  lpruns[1].append(r*10000+p)
      return lpruns
 

◆ makeHistos()

runProd.prodManager.makeHistos	(	self,
		runList
	)

Definition at line 346 of file runProd.py.

346 def makeHistos(self,runList):

◆ mips()

runProd.prodManager.mips ( self )

Definition at line 354 of file runProd.py.

354 def mips(self):

◆ rawStats()

runProd.prodManager.rawStats	(	self,
		runList
	)

Definition at line 338 of file runProd.py.

   def rawStats(self,runList):
      for run in runList:
         runNr   = str(run).zfill(6)
         r = ROOT.TFile.Open(os.environ['EOSSHIP']+path+"/run_"+runNr+"/data.root")
         if fI.Get('event'): raw = r.Get("event").GetEntries()
         else:               raw = r.Get("data").GetEntries()
         print(run,':',raw)
 

◆ RerunDataQuality()

runProd.prodManager.RerunDataQuality	(	self,
		runNrs = `[]`,
		rMin = `-1`,
		rMax = `99999`
	)

Definition at line 137 of file runProd.py.

   def RerunDataQuality(self,runNrs=[],rMin=-1,rMax=99999):
      monitorCommand = "python $SNDSW_ROOT/shipLHC/scripts/run_Monitoring.py -r XXXX --server=$EOSSHIP \
                        -b 100000 -p "+pathConv+" -g GGGG "\
                        +" --postScale "+str(options.postScale)+ " --ScifiResUnbiased 1 --batch --sudo "
      if options.parallel>1: monitorCommand += " --postScale "+str(options.parallel)
      if len(runNrs) ==0:
         self.getRunNrFromOffline(rMin,rMax)
         runNrs = self.dqDataFiles
      for r in runNrs:
           print('executing DQ for run %i'%(r))
           if   r  < 4575:  geoFile =  "../geofile_sndlhc_TI18_V3_08August2022.root"
           elif r  < 4855:   geoFile =  "../geofile_sndlhc_TI18_V5_14August2022.root"
           elif r  < 5172:  geoFile =  "../geofile_sndlhc_TI18_V6_08October2022.root"
           elif r  < 5485: geoFile =  "../geofile_sndlhc_TI18_V7_22November2022.root"
           elif r  < 7357: geoFile =  "../geofile_sndlhc_TI18_V1_2023.root"
           else: geoFile =  options.geofile
           os.system(monitorCommand.replace('XXXX',str(r)).replace('GGGG',geoFile)+" &")
           time.sleep(20)
           while self.count_python_processes('run_Monitoring')>(ncpus-5) or psutil.virtual_memory()[2]>90 : time.sleep(300)
 

◆ runDataQuality()

runProd.prodManager.runDataQuality	(	self,
		latest
	)

Definition at line 93 of file runProd.py.

   def runDataQuality(self,latest):
      monitorCommand = "python $SNDSW_ROOT/shipLHC/scripts/run_Monitoring.py -r XXXX --server=$EOSSHIP \
                        -b 100000 -p "+pathConv+" -praw "+path+" -g GGGG "\
                        +" --postScale "+str(options.postScale)+ " --ScifiResUnbiased 1 --batch --sudo  "
      if options.parallel>1: monitorCommand += " --parallel "+str(options.parallel)
      convDataFiles = self.getFileList(pathConv,latest,options.rMax,minSize=0)
      self.checkEOS(copy=False,latest=latest)
      # remove directories which are not completely copied
      for r in self.missing:
             if r in convDataFiles: convDataFiles.pop(r)
      # remove directories which are not fully converted
      rawDataFiles = self.getFileList(path,latest,options.rMax,minSize=10E6)
      self.RawrunNrs = {}
      for x in rawDataFiles:
             r =  x//10000
             if not r in self.RawrunNrs: self.RawrunNrs[r] = []
             self.RawrunNrs[r].append(x)
 
      orderedCDF = list(convDataFiles.keys())
      lpruns = self.list_of_runs('run_Monitoring')
 
      self.getRunNrFromOffline(latest)
      self.runNrs = {}
      for x in orderedCDF:
          r = x//10000
          if  r in self.runNrs:
              self.runNrs[r].append(x)
              continue
          if r in lpruns[0]: continue
          if r in self.dqDataFiles: continue
          self.runNrs[r] = [x]
          
      for r in self.runNrs:
           if r > options.rMax or r < options.rMin: continue # outside range
           if not r in self.RawrunNrs: 
               print('run not found in raw data',r)
               continue # file converted but not enough events
           if len(self.runNrs[r]) != len(self.RawrunNrs[r]): 
               print('run not complete',r)
               continue  # not all files converted.
           print('executing DQ for run %i'%(r))
           os.system(monitorCommand.replace('XXXX',str(r)).replace('GGGG',options.geofile)+" &")
           while self.count_python_processes('run_Monitoring')>(ncpus-2) or psutil.virtual_memory()[2]>90 : time.sleep(1800)
 

◆ runSinglePartition()

runProd.prodManager.runSinglePartition	(	self,
		path,
		r,
		p,
		EOScopy = `False`,
		check = `True`
	)

Definition at line 174 of file runProd.py.

   def runSinglePartition(self,path,r,p,EOScopy=False,check=True):
       while self.count_python_processes('convertRawData')>ncpus or self.count_python_processes('runProd')>ncpus: time.sleep(300)
       process = []
       pid = 0
       try:
           pid = os.fork()
       except OSError:
           print("Could not create a child process")
       if pid!=0:
          process.append(pid)
       else:          
         inFile = self.options.server+path+'run_'+ r+'/data_'+p+'.root'
         fI = ROOT.TFile.Open(inFile)
         if not fI:
           print('file not found',path,r,p)
           exit()
         if not fI.Get('event') and not fI.Get('data'):
           print('file corrupted',path,r,p)
           exit()
         command =   "python $SNDSW_ROOT/shipLHC/rawData/convertRawData.py  -r "+str(int(r))+ " -b 1000 -p "+path+" --server="+self.options.server
         if options.FairTask_convRaw:  command+= " -cpp "
         command += " -P "+str(int(p)) + " >log_"+r+'-'+p    
         print("execute ",command)
         os.system(command)
         if check:
          rc = self.checkFile(path,r,p)
          if rc<0: 
             print('converting failed',r,p,rc)
             exit()
         print('finished converting ',r,p)
         tmp = {int(r):[int(p)]}
         if EOScopy:  self.copy2EOS(path,tmp,self.options.overwrite)
         exit(0)
 

Member Data Documentation

◆ command

str runProd.prodManager.command = "$SNDSW_ROOT/shipLHC/scripts/Survey-MufiScifi.py -r "+str(run)+" -p "+pathConv+" -g geofile_sndlhc_H6.root -c Mufi_Efficiency -n -1 -t DS"

static

Definition at line 349 of file runProd.py.

◆ daqInventory

runProd.prodManager.daqInventory

Definition at line 312 of file runProd.py.

◆ dqDataFiles

runProd.prodManager.dqDataFiles

Definition at line 85 of file runProd.py.

◆ eosInventory

runProd.prodManager.eosInventory

Definition at line 309 of file runProd.py.

◆ Mext

runProd.prodManager.Mext

Definition at line 32 of file runProd.py.

◆ missing

runProd.prodManager.missing

Definition at line 314 of file runProd.py.

◆ options

runProd.prodManager.options

Definition at line 35 of file runProd.py.

◆ RawrunNrs

runProd.prodManager.RawrunNrs

Definition at line 105 of file runProd.py.

◆ runNrs

runProd.prodManager.runNrs

Definition at line 36 of file runProd.py.

The documentation for this class was generated from the following file:

shipLHC/rawData/runProd.py

Public Member Functions

Public Attributes

Static Public Attributes