fix for closeOut for not using oldGlobalMonitor no more

Author: efajardo

closeOutWorkflows.py

@@ -59,6 +59,19 @@ def getOverviewRequest():
         requests = json.loads(r2.read())
 	return requests
 
+def getOverviewRequestsWMStats(url):
+	conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'),
+                                     key_file = os.getenv('X509_USER_PROXY'))
+	conn.request("GET",
+                 "/couchdb/wmstats/_design/WMStats/_view/requestByStatusAndType?stale=update_after")
+	response = conn.getresponse()
+	data = response.read()
+	conn.close()
+	myString=data.decode('utf-8')
+	workflows=json.loads(myString)['rows']
+	return workflows
+
+
 #Returns a list with the custodial sites for a request
 def findCustodial(url, requestname):
 	custodialSites=[]
@@ -80,16 +93,21 @@ def findCustodial(url, requestname):
 def classifyCompletedRequests(url, requests):
 	workflows={'ReDigi':[],'MonteCarloFromGEN':[],'MonteCarlo':[] , 'ReReco':[], 'LHEStepZero':[]}
 	for request in requests:
-	    name=request['request_name']
-	    status='NoStatus'
-	    if 'status' in request.keys():
-			status=request['status']
-            requestType='NoType'
-	    if 'type' in request.keys():
-			 requestType=request['type']
+	    name=request['id']
+	    if len(request['key'])<3:
+		print request
+		continue
+	    status=request['key'][1]
+	    requestType=request['key'][2]
 	    if status=='completed':
-		if requestType=='MonteCarloFromGEN' or requestType=='MonteCarlo' or requestType=='LHEStepZero'or requestType=='ReDigi' or requestType=='ReReco':
-			workflows[requestType].append(name)
+		if requestType=='MonteCarlo':
+			datasets=phedexSubscription.outputdatasetsWorkflow(url, name)
+			if 'GEN' in datasets[0]:
+				workflows['LHEStepZero'].append(name)
+			else:
+				workflows[requestType].append(name)
+		if requestType=='MonteCarloFromGEN' or requestType=='LHEStepZero'or requestType=='ReDigi' or requestType=='ReReco':
+				workflows[requestType].append(name)
 	return workflows
 
 def testOutputDataset(datasetName):
@@ -254,7 +272,7 @@ def PercentageCompletion(url, workflow, dataset):
 def main():
 	url='cmsweb.cern.ch'
 	print "Gathering Requests"
-	requests=getOverviewRequest()
+	requests=getOverviewRequestsWMStats(url)
 	print "Classifying Requests"
 	workflowsCompleted=classifyCompletedRequests(url, requests)
 	print '-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------'

dbsTest.py~

@@ -4,8 +4,8 @@ import urllib2,urllib, httplib, sys, re, os, json, phedexSubscription
 from xml.dom.minidom import getDOMImplementation
 from das_client import get_data
 #das_host='https://das.cern.ch'
-das_host='https://cmsweb.cern.ch'
-#das_host='https://cmsweb-testbed.cern.ch'
+#das_host='https://cmsweb.cern.ch'
+das_host='https://cmsweb-testbed.cern.ch'
 #das_host='https://das-dbs3.cern.ch'
 
 def getWorkflowType(url, workflow):
@@ -177,7 +177,7 @@ def getEventCountDataSet(das_url, dataset):
     else:
         result = das_data
     if result['status'] == 'fail' :
-        print 'DAS query failed with reason:',result['reason']
+        print 'DAS query' + query+' failed with reason:',result['reason']
     else:
 	if len(result['data'])==0:#dataset not yet registered in DBS
 		return 0
@@ -300,7 +300,8 @@ def getEventsBlock(das_url, block_name):
     else:
         result = das_data
     if result['status'] == 'fail' :
-        print 'DAS query failed with reason:',result['reason']
+        print 'DAS quert ', query	
+	print 'failed with reason:',result['reason']
     else:
 	if len(result['data'])==0:#dataset not yet registered in DBS
 		return 0
@@ -408,15 +409,13 @@ def getInputEvents(url, workflow):
     if requestType == 'TaskChain':
         return handleTaskChain(request)
     #In case some parameters miss in the request like blockwhitelist, blockblack list and so on or it was injected as a string.
-    BlockWhitelist=[]
-    BlockBlacklist=[]
-    runWhitelist=[]
-    runBlacklist=[]
     for listitem in ["RunWhitelist", "RunBlacklist", "BlockWhitelist",
                            "BlockBlacklist"]:
 	if listitem in request:
 		if request[listitem]=='[]':
 			request[listitem]=[]
+		if type(request[listitem]) is not list:#if there is not a list but just one element we convert it to a list with one element.
+			request[listitem]=re.split(r",",request[listitem])
 	else:
 		request[listitem]=[]
     inputDataSet=request['InputDataset']
@@ -430,12 +429,12 @@ def getInputEvents(url, workflow):
         else:
             return getRunLumiCountDataset(das_host, request['InputDataset'])
     events=getEventCountDataSet(das_host, request['InputDataset'])
-    if len(BlockBlacklist)>0:
-        events=events-EventsBlockList(request['InputDataset'], request['BlockBlacklist'])
-    if len(runWhitelist)>0:
-        events=EventsRunList(das_host, request['InputDataset'], request['RunWhitelist'])
-    if len(BlockWhitelist)>0:
-        events=EventsBlockList(das_host, request['InputDataset'], request['BlockWhitelist'])
+    if len(request['BlockBlacklist'])>0:
+	events=events-EventsBlockList(request['InputDataset'], request['BlockBlacklist'])
+    if len(request['RunWhitelist'])>0:
+	events=EventsRunList(das_host, request['InputDataset'], request['RunWhitelist'])
+    if len(request['BlockWhitelist'])>0:
+	events=EventsBlockList(das_host, request['InputDataset'], request['BlockWhitelist'])
     if 'FilterEfficiency' in request.keys():
         return float(request['FilterEfficiency'])*events
     else:

notCustodialDatasets.py

@@ -90,7 +90,7 @@ def getBlocksProducedLastWeek(url):
 
 def getdatasetsProducedLastWeek(url):
 	datasetsWeek=[]
-	oneWeekTime=time.time()-7*24*60*60
+	oneWeekTime=time.time()-30*24*60*60
         #oneWeekTime=time.time()-3*30*24*60*60
 	conn  =  httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
 	r1=conn.request("GET",'/phedex/datasvc/json/prod/data?create_since='+str(oneWeekTime)+'&dataset=/*/*/*SIM&level=block')

resubmit.py

@@ -82,6 +82,10 @@ def retrieveSchema(workflowName, user, group ):
             continue
 	elif value != None:
             schema[key] = value
+    if 'LumisPerJob' not in schema and schema['RequestType']=='MonteCarlo':
+	schema['LumisPerJob']=300
+    if 'EventsPerJob' not in schema and schema['RequestType']=='MonteCarlo':
+	schema['EventsPerJob']=120000
     return schema
 
 def submitWorkflow(schema):