diff --git a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst index 6f85a328857..2652bd93958 100644 --- a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst +++ b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst @@ -84,7 +84,7 @@ The given periods above are also the default periods in the code. Enable WMSHistory monitoring ============================ -You have to add ``Monitoring`` to the ``Backends`` option of WorkloadManagemet/StatesAccountingAgent. +You have to add ``Monitoring`` to the ``Backends`` option of WorkloadManagement/StatesAccountingAgent. If you do so, this agent will collect information using the JobDB and send it to the Elasticsearch database. This same agent can also report to the MySQL backend of the Accounting system (which is in fact the default). @@ -112,11 +112,12 @@ You can configure the MQ in the local dirac.cfg file where the agent is running: } } -A dashboard for WMSHistory monitoring ``WMSDashboard`` is available `here `__ for import both as a JSON file and as a NDJSON (as support for JSON is being removed in the latest versions of Kibana). -The dashboard is not compatible with older versions of ElasticSearch (such as ES6). -To import it in the Kibana UI, go to Management -> Saved Objects -> Import and import the JSON file. +*Kibana dashboard for WMSHistory* + A dashboard for WMSHistory monitoring ``WMSDashboard`` is available `here `__ for import both as a JSON file and as a NDJSON (as support for JSON is being removed in the latest versions of Kibana). + The dashboard is not compatible with older versions of ElasticSearch (such as ES6). + To import it in the Kibana UI, go to Management -> Saved Objects -> Import and import the JSON file. -Note: the JSON file already contains the index patterns needed for the visualizations. You may need to adapt the index patterns to your existing ones. + Note: the JSON file already contains the index patterns needed for the visualizations. You may need to adapt the index patterns to your existing ones. Enable Component monitoring @@ -153,7 +154,11 @@ In order to enable RMSMonitoring we need to set value of ``EnableRMSMonitoring`` } } +Enable Pilot Submission Monitoring +================================== +In order to enable the monitoring of the pilot submission so that they will be sent to ES backend (by default they are sent to Accounting), you need to set +``sendPilotSubmissionMonitoring = True`` for this option in WorkloadManagement/SiteDirector. Accessing the Monitoring information ===================================== diff --git a/docs/source/DeveloperGuide/Systems/Monitoring/index.rst b/docs/source/DeveloperGuide/Systems/Monitoring/index.rst index 68fe5f6f397..1ee212866d5 100644 --- a/docs/source/DeveloperGuide/Systems/Monitoring/index.rst +++ b/docs/source/DeveloperGuide/Systems/Monitoring/index.rst @@ -55,11 +55,12 @@ A new monitoring type can be added: self.setKeyFields( ['cond1', 'cond2'] ) self.setMonitoringFields( [ 'ex1' ] ) - - create the plotter: MonitoringSystem/Client/private/Plotters/ExamplePlotter.py + - create the plotter: MonitoringSystem/private/Plotters/ExamplePlotter.py Note: The file name must ends with Plotter word. You have to implement two functions: def _reportExample( self, reportRequest ): + def _plotExample( self, reportRequest, plotInfo, filename ): In the Monitoring page you will see and Example. But if you want to rename it: diff --git a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py index 222ce238dc0..0c1fd69d315 100644 --- a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py +++ b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py @@ -15,11 +15,6 @@ a MQ service is available, if the MQ is not working a failover will be performed. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" import threading import json diff --git a/src/DIRAC/MonitoringSystem/Client/Types/PilotMonitoring.py b/src/DIRAC/MonitoringSystem/Client/Types/PilotMonitoring.py new file mode 100644 index 00000000000..c6fde6614d6 --- /dev/null +++ b/src/DIRAC/MonitoringSystem/Client/Types/PilotMonitoring.py @@ -0,0 +1,30 @@ +""" +Monitoring Type for Pilot Submission +""" + +from DIRAC.MonitoringSystem.Client.Types.BaseType import BaseType + + +class PilotMonitoring(BaseType): + def __init__(self): + + super().__init__() + + self.keyFields = ["HostName", "SiteDirector", "Site", "CE", "Queue", "Status"] + + self.monitoringFields = ["NumTotal", "NumSucceeded"] + + self.index = "pilotstats_index" + + self.addMapping( + { + "HostName": {"type": "keyword"}, + "SiteDirector": {"type": "keyword"}, + "Site": {"type": "keyword"}, + "CE": {"type": "keyword"}, + "Queue": {"type": "keyword"}, + "Status": {"type": "keyword"}, + } + ) + + self.checkType() diff --git a/src/DIRAC/MonitoringSystem/Client/Types/WMSHistory.py b/src/DIRAC/MonitoringSystem/Client/Types/WMSHistory.py index afe7994e454..6113b38b63d 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/WMSHistory.py @@ -3,9 +3,6 @@ Filled by the agent "WorkloadManagement/StatesAccountingAgent" """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from DIRAC.MonitoringSystem.Client.Types.BaseType import BaseType diff --git a/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py b/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py index e159edd2dd6..fa21178582d 100644 --- a/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py +++ b/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py @@ -32,11 +32,6 @@ """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" import time import calendar diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/PilotMonitoringPlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/PilotMonitoringPlotter.py new file mode 100644 index 00000000000..616543cf4aa --- /dev/null +++ b/src/DIRAC/MonitoringSystem/private/Plotters/PilotMonitoringPlotter.py @@ -0,0 +1,111 @@ +""" +This class is used to define the plot using the plot attributes. +""" + +from DIRAC import S_OK + +from DIRAC.MonitoringSystem.Client.Types.PilotMonitoring import PilotMonitoring +from DIRAC.MonitoringSystem.private.Plotters.BasePlotter import BasePlotter + + +class WMSHistoryPlotter(BasePlotter): + + """ + .. class:: PilotMonitoringPlotter + + It is used to crate the plots. + + param: str _typeName monitoring type + param: list _typeKeyFields list of keys what we monitor (list of attributes) + """ + + _typeName = "PilotMonitoring" + _typeKeyFields = PilotMonitoring().keyFields + + def reportNumberOfSubmissions(self, reportRequest): + """It is used to retrieve the data from the database. + + :param dict reportRequest: contains attributes used to create the plot. + :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length + """ + retVal = self._getTimedData( + startTime=reportRequest["startTime"], + endTime=reportRequest["endTime"], + selectField="NumTotal", + preCondDict=reportRequest["condDict"], + metadataDict=None, + ) + if not retVal["OK"]: + return retVal + dataDict, granularity = retVal["Value"] + return S_OK({"data": dataDict, "granularity": granularity}) + + def _plotNumberOfSubmissions(self, reportRequest, plotInfo, filename): + """It creates the plot. + + :param dict reportRequest: plot attributes + :param dict plotInfo: contains all the data which are used to create the plot + :param str filename: + :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE + """ + metadata = { + "title": "Pilot Submissions by %s" % reportRequest["grouping"], + "starttime": reportRequest["startTime"], + "endtime": reportRequest["endTime"], + "span": plotInfo["granularity"], + "skipEdgeColor": True, + "ylabel": "Submissions", + } + + plotInfo["data"] = self._fillWithZero( + granularity=plotInfo["granularity"], + startEpoch=reportRequest["startTime"], + endEpoch=reportRequest["endTime"], + dataDict=plotInfo["data"], + ) + + return self._generateStackedLinePlot(filename=filename, dataDict=plotInfo["data"], metadata=metadata) + + def reportNumSucceeded(self, reportRequest): + """It is used to retrieve the data from the database. + + :param dict reportRequest: contains attributes used to create the plot. + :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length + """ + retVal = self._getTimedData( + startTime=reportRequest["startTime"], + endTime=reportRequest["endTime"], + selectField="NumSucceeded", + preCondDict=reportRequest["condDict"], + metadataDict=None, + ) + if not retVal["OK"]: + return retVal + dataDict, granularity = retVal["Value"] + return S_OK({"data": dataDict, "granularity": granularity}) + + def _plotNumSucceeded(self, reportRequest, plotInfo, filename): + """It creates the plot. + + :param dict reportRequest: plot attributes + :param dict plotInfo: contains all the data which are used to create the plot + :param str filename: + :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE + """ + metadata = { + "title": "SuSubmissions by %s" % reportRequest["grouping"], + "starttime": reportRequest["startTime"], + "endtime": reportRequest["endTime"], + "span": plotInfo["granularity"], + "skipEdgeColor": True, + "ylabel": "submissions", + } + + plotInfo["data"] = self._fillWithZero( + granularity=plotInfo["granularity"], + startEpoch=reportRequest["startTime"], + endEpoch=reportRequest["endTime"], + dataDict=plotInfo["data"], + ) + + return self._generateStackedLinePlot(filename=filename, dataDict=plotInfo["data"], metadata=metadata) diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/WMSHistoryPlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/WMSHistoryPlotter.py index 87d04b304b7..ab7bc643f46 100644 --- a/src/DIRAC/MonitoringSystem/private/Plotters/WMSHistoryPlotter.py +++ b/src/DIRAC/MonitoringSystem/private/Plotters/WMSHistoryPlotter.py @@ -1,17 +1,12 @@ """ This class is used to define the plot using the plot attributes. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from DIRAC import S_OK from DIRAC.MonitoringSystem.Client.Types.WMSHistory import WMSHistory from DIRAC.MonitoringSystem.private.Plotters.BasePlotter import BasePlotter -__RCSID__ = "$Id$" - class WMSHistoryPlotter(BasePlotter): diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 8bd588bedf5..2c6f8cf33bc 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -15,16 +15,18 @@ from concurrent.futures import ThreadPoolExecutor import DIRAC -from DIRAC import S_OK, gConfig +from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.Core.Base.AgentModule import AgentModule -from DIRAC.Core.Utilities.Time import dateTime, second +from DIRAC.Core.Utilities.Time import dateTime, second, toEpoch from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.AccountingSystem.Client.Types.Pilot import Pilot as PilotAccounting from DIRAC.AccountingSystem.Client.Types.PilotSubmission import PilotSubmission as PilotSubmissionAccounting +from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter +from DIRAC.MonitoringSystem.Client.Types.PilotMonitoring import PilotMonitoring as PilotSubmissionMonitoring from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient from DIRAC.WorkloadManagementSystem.Client import PilotStatus from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient @@ -87,7 +89,7 @@ def __init__(self, *args, **kwargs): self.getOutput = False self.sendAccounting = True self.sendSubmissionAccounting = True - + self.sendSubmissionMonitoring = False self.siteClient = None self.rssClient = None self.rssFlag = None @@ -192,6 +194,9 @@ def beginExecution(self): self.sendSubmissionAccounting = self.am_getOption( "SendPilotSubmissionAccounting", self.sendSubmissionAccounting ) + self.sendSubmissionMonitoring = self.am_getOption( + "SendPilotSubmissionMonitoring", self.sendSubmissionMonitoring + ) # Get the site description dictionary siteNames = None @@ -270,6 +275,8 @@ def beginExecution(self): self.log.always("Pilot accounting sending requested") if self.sendSubmissionAccounting: self.log.always("Pilot submission accounting sending requested") + if self.sendSubmissionMonitoring: + self.log.always("Pilot submission monitoring sending requested") self.log.always("MaxPilotsToSubmit:", self.maxPilotsToSubmit) @@ -716,7 +723,15 @@ def _submitPilotsToQueue(self, pilotsToSubmit, ce, queue): 0, "Failed", ) - + if self.sendSubmissionMonitoring: + self.sendPilotSubmissionMonitoring( + self.queueDict[queue]["Site"], + self.queueDict[queue]["CEName"], + self.queueDict[queue]["QueueName"], + pilotsToSubmit, + 0, + "Failed", + ) self.failedQueues[queue] += 1 return submitResult @@ -739,6 +754,15 @@ def _submitPilotsToQueue(self, pilotsToSubmit, ce, queue): len(pilotList), "Succeeded", ) + if self.sendSubmissionMonitoring: + self.sendPilotSubmissionMonitoring( + self.queueDict[queue]["Site"], + self.queueDict[queue]["CEName"], + self.queueDict[queue]["QueueName"], + len(pilotList), + len(pilotList), + "Succeeded", + ) return S_OK((pilotList, stampDict)) @@ -1359,3 +1383,46 @@ def sendPilotSubmissionAccounting(self, siteName, ceName, queueName, numTotal, n if not result["OK"]: self.log.error("Error in Commit:" + result["Message"]) return result + + def sendPilotSubmissionMonitoring(self, siteName, ceName, queueName, numTotal, numSucceeded, status): + """Sends pilot submission records to monitoring + + :param str siteName: Site name + :param str ceName: CE name + :param str queueName: queue Name + :param int numTotal: Total number of submission + :param int numSucceeded: Total number of submission succeeded + :param str status: 'Succeeded' or 'Failed' + + :returns: S_OK / S_ERROR + """ + + pilotMonitoringReporter = MonitoringReporter(monitoringType="PilotMonitoring") + + if hasattr(self, "_AgentModule__moduleProperties"): + siteDirName = self.am_getModuleParam("agentName") + else: # In case it is not executed as agent + siteDirName = "Client" + + pilotMonitoringData = [ + { + "HostName": DIRAC.siteName(), + "SiteDirector": siteDirName, + "Site": siteName, + "CE": ceName, + "Queue": queueName, + "Status": status, + "NumTotal": numTotal, + "NumSucceded": numSucceeded, + "timestamp": int(toEpoch(dateTime())), + } + ] + pilotMonitoringReporter.addRecord(pilotMonitoringData) + result = pilotMonitoringReporter.commit() + + self.log.verbose("Committing pilot submission to monitoring") + if not result["OK"]: + self.log.error("Couldn't commit pilot submission to monitoring", result["Message"]) + return S_ERROR() + self.log.verbose("Done committing to monitoring") + return S_OK() diff --git a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg index fcdb4ffb567..a216048b6b9 100644 --- a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg @@ -277,6 +277,8 @@ Agents SendPilotAccounting = True # Boolean value that indicates if the pilot submission statistics will be sended for accounting SendPilotSubmissionAccounting = True + # Boolean value that indicates if the pilot submission statistics will be sended for monitoring + SendPilotSubmissionMonitoring = False } ##END ##BEGIN StatesAccountingAgent diff --git a/tests/Integration/Monitoring/Test_MonitoringDB.py b/tests/Integration/Monitoring/Test_MonitoringDB.py index c4ff2fe8c96..de29aac648a 100644 --- a/tests/Integration/Monitoring/Test_MonitoringDB.py +++ b/tests/Integration/Monitoring/Test_MonitoringDB.py @@ -1,12 +1,6 @@ """ Test for MonitoringDB """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - import time import json import pytest diff --git a/tests/Integration/Monitoring/Test_MonitoringReporter.py b/tests/Integration/Monitoring/Test_MonitoringReporter.py index 8fe3797948c..dd20e3fbc8f 100644 --- a/tests/Integration/Monitoring/Test_MonitoringReporter.py +++ b/tests/Integration/Monitoring/Test_MonitoringReporter.py @@ -54,813 +54,830 @@ # pylint: disable=invalid-name,wrong-import-position -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import unittest import sys +import pytest from DIRAC.Core.Utilities.DIRACScript import DIRACScript as Script Script.parseCommandLine() from DIRAC import gLogger - from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter +gLogger.setLevel("INFO") -class MonitoringTestCase(unittest.TestCase): - def setUp(self): - gLogger.setLevel("INFO") - - self.wmsMonitoringReporter = MonitoringReporter(monitoringType="WMSHistory") - self.componentMonitoringReporter = MonitoringReporter(monitoringType="ComponentMonitoring") +wmsMonitoringReporter = MonitoringReporter(monitoringType="WMSHistory") +componentMonitoringReporter = MonitoringReporter(monitoringType="ComponentMonitoring") +pilotMonitoringReporter = MonitoringReporter(monitoringType="PilotMonitoring") - self.data = [ - { - "Status": "Waiting", - "Jobs": 2, - "timestamp": 1458130176, - "JobSplitType": "MCStripping", - "MinorStatus": "unset", - "Site": "LCG.GRIDKA.de", - "Reschedules": 0, - "ApplicationStatus": "unset", - "User": "phicharp", - "JobGroup": "00049848", - "UserGroup": "lhcb_mc", - "metric": "WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458130176, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"LCG.PIC.es", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"olupton", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458130176, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"olupton", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458130176, - u"JobSplitType": u"MCStripping", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049845", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 34, - u"timestamp": 1458141578, - u"JobSplitType": u"DataStripping", - u"MinorStatus": u"unset", - u"Site": u"Group.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050299", - u"UserGroup": u"lhcb_data", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 120, - u"timestamp": 1458141578, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"LCG.CERN.ch", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"mvesteri", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458141578, - u"JobSplitType": u"MCStripping", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049845", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 2, - u"timestamp": 1458141578, - u"JobSplitType": u"MCStripping", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049848", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458141578, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050286", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 95, - u"timestamp": 1458199202, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"Multiple", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"mamartin", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 3, - u"timestamp": 1458199202, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"Multiple", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"olupton", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 129, - u"timestamp": 1458199202, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"Multiple", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049844", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 5, - u"timestamp": 1458217812, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.IHEP.su", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050232", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 7, - u"timestamp": 1458217812, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.IHEP.su", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050234", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458217812, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.IHEP.su", - u"Reschedules": 1, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050236", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 3, - u"timestamp": 1458217812, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.IHEP.su", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050238", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 2, - u"timestamp": 1458217812, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.IHEP.su", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050248", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 12, - u"timestamp": 1458218413, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050248", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 5, - u"timestamp": 1458218413, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050250", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 4, - u"timestamp": 1458218413, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050251", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458218413, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.CNAF.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050280", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 24, - u"timestamp": 1458219012, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.NIKHEF.nl", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050248", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 3, - u"timestamp": 1458219012, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.NIKHEF.nl", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050251", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458222013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bologna.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050303", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 7, - u"timestamp": 1458222013, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bristol.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"clangenb", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 2, - u"timestamp": 1458222013, - u"JobSplitType": u"User", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bristol.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"mrwillia", - u"JobGroup": u"lhcb", - u"UserGroup": u"lhcb_user", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458222013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bari.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050244", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 11, - u"timestamp": 1458222013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bari.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050246", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 22, - u"timestamp": 1458222013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.Bari.it", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050248", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 23, - u"timestamp": 1458225013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.DESYZN.de", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049844", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 18, - u"timestamp": 1458225013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.DESYZN.de", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00049847", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458225013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.DESYZN.de", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050238", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Running", - "Jobs": 1, - u"timestamp": 1458225013, - u"JobSplitType": u"MCSimulation", - u"MinorStatus": u"unset", - u"Site": u"LCG.DESYZN.de", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050246", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RRCKI.ru", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050243", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RRCKI.ru", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050251", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCStripping", - u"MinorStatus": u"unset", - u"Site": u"LCG.RRCKI.ru", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050256", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050229", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050241", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 1, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050243", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - { - u"Status": u"Waiting", - "Jobs": 2, - u"timestamp": 1458226213, - u"JobSplitType": u"MCReconstruction", - u"MinorStatus": u"unset", - u"Site": u"LCG.RAL.uk", - u"Reschedules": 0, - u"ApplicationStatus": u"unset", - u"User": u"phicharp", - u"JobGroup": u"00050247", - u"UserGroup": u"lhcb_mc", - u"metric": u"WMSHistory", - }, - ] +data = [ + { + "Status": "Waiting", + "Jobs": 2, + "timestamp": 1458130176, + "JobSplitType": "MCStripping", + "MinorStatus": "unset", + "Site": "LCG.GRIDKA.de", + "Reschedules": 0, + "ApplicationStatus": "unset", + "User": "phicharp", + "JobGroup": "00049848", + "UserGroup": "lhcb_mc", + "metric": "WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458130176, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"LCG.PIC.es", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"olupton", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458130176, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"olupton", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458130176, + u"JobSplitType": u"MCStripping", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049845", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 34, + u"timestamp": 1458141578, + u"JobSplitType": u"DataStripping", + u"MinorStatus": u"unset", + u"Site": u"Group.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050299", + u"UserGroup": u"lhcb_data", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 120, + u"timestamp": 1458141578, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"LCG.CERN.ch", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"mvesteri", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458141578, + u"JobSplitType": u"MCStripping", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049845", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 2, + u"timestamp": 1458141578, + u"JobSplitType": u"MCStripping", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049848", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458141578, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050286", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 95, + u"timestamp": 1458199202, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"Multiple", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"mamartin", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 3, + u"timestamp": 1458199202, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"Multiple", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"olupton", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 129, + u"timestamp": 1458199202, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"Multiple", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049844", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 5, + u"timestamp": 1458217812, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.IHEP.su", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050232", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 7, + u"timestamp": 1458217812, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.IHEP.su", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050234", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458217812, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.IHEP.su", + u"Reschedules": 1, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050236", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 3, + u"timestamp": 1458217812, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.IHEP.su", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050238", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 2, + u"timestamp": 1458217812, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.IHEP.su", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050248", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 12, + u"timestamp": 1458218413, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050248", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 5, + u"timestamp": 1458218413, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050250", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 4, + u"timestamp": 1458218413, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050251", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458218413, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.CNAF.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050280", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 24, + u"timestamp": 1458219012, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.NIKHEF.nl", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050248", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 3, + u"timestamp": 1458219012, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.NIKHEF.nl", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050251", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458222013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bologna.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050303", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 7, + u"timestamp": 1458222013, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bristol.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"clangenb", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 2, + u"timestamp": 1458222013, + u"JobSplitType": u"User", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bristol.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"mrwillia", + u"JobGroup": u"lhcb", + u"UserGroup": u"lhcb_user", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458222013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bari.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050244", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 11, + u"timestamp": 1458222013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bari.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050246", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 22, + u"timestamp": 1458222013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.Bari.it", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050248", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 23, + u"timestamp": 1458225013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.DESYZN.de", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049844", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 18, + u"timestamp": 1458225013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.DESYZN.de", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00049847", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458225013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.DESYZN.de", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050238", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Running", + "Jobs": 1, + u"timestamp": 1458225013, + u"JobSplitType": u"MCSimulation", + u"MinorStatus": u"unset", + u"Site": u"LCG.DESYZN.de", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050246", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RRCKI.ru", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050243", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RRCKI.ru", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050251", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCStripping", + u"MinorStatus": u"unset", + u"Site": u"LCG.RRCKI.ru", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050256", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050229", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050241", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 1, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050243", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, + { + u"Status": u"Waiting", + "Jobs": 2, + u"timestamp": 1458226213, + u"JobSplitType": u"MCReconstruction", + u"MinorStatus": u"unset", + u"Site": u"LCG.RAL.uk", + u"Reschedules": 0, + u"ApplicationStatus": u"unset", + u"User": u"phicharp", + u"JobGroup": u"00050247", + u"UserGroup": u"lhcb_mc", + u"metric": u"WMSHistory", + }, +] - # This dataset is used for the ComponentMonitoringType as the data which gets stored in this type - # is usually with these type of fields. - self.activityMonitoringData = [ - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - { - u"site": u"dirac-dev", - "componentType": u"service", - u"component": u"Framework_SystemAdministrator", - u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", - u"Connections": 92946, - u"Queries": 1880, - u"PendingQueries": 200, - u"ActiveQueries": 200, - u"RunningThreads": 200, - u"MaxFD": 200, - u"timestamp": 1458226213, - }, - ] +# This dataset is used for the ComponentMonitoringType as the data which gets stored in this type +# is usually with these type of fields. +activityMonitoringData = [ + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, + { + u"site": u"dirac-dev", + "componentType": u"service", + u"component": u"Framework_SystemAdministrator", + u"componentLocation": u"dips://dirac-dev:9162/Framework/SystemAdministrator", + u"Connections": 92946, + u"Queries": 1880, + u"PendingQueries": 200, + u"ActiveQueries": 200, + u"RunningThreads": 200, + u"MaxFD": 200, + u"timestamp": 1458226213, + }, +] +# Test data for the PilotMonitoring type +pilotMonitoringData = [ + { + "HostName": "lhcbvoboxcertif00.cern.ch", + "SiteDirector": "SiteDirector-CERN", + "Site": "LCG.CERN.cern", + "CE": "ce503.cern.ch", + "Queue": "ce503.cern.ch:ce503.cern.ch-condor", + "Status": "failed", + "NumTotal": "10", + "NumSucceded": "3", + "timestamp": 1458226213, + }, + { + "HostName": "lhcbvoboxcertif00.cern.ch", + "SiteDirector": "SiteDirector-CNAF", + "Site": "LCG.CNAF.it", + "CE": "ce504.cern.ch", + "Queue": "ce504.cern.ch:ce504.cern.ch-condor", + "Status": "failed", + "NumTotal": "7", + "NumSucceded": "1", + "timestamp": 1458226213, + }, +] - def tearDown(self): - pass +def test_addWMSRecords(): + for record in data: + wmsMonitoringReporter.addRecord(record) + result = wmsMonitoringReporter.commit() + assert result["OK"] + assert result["Value"] == len(data) -class MonitoringReporterAdd(MonitoringTestCase): - def test_addWMSRecords(self): - for record in self.data: - self.wmsMonitoringReporter.addRecord(record) - result = self.wmsMonitoringReporter.commit() - self.assertTrue(result["OK"]) - self.assertEqual(result["Value"], len(self.data)) - def test_addComponentRecords(self): - for record in self.activityMonitoringData: - self.componentMonitoringReporter.addRecord(record) - result = self.componentMonitoringReporter.commit() - self.assertTrue(result["OK"]) - self.assertEqual(result["Value"], len(self.activityMonitoringData)) +def test_addComponentRecords(): + for record in activityMonitoringData: + componentMonitoringReporter.addRecord(record) + result = componentMonitoringReporter.commit() + assert result["OK"] + assert result["Value"] == len(activityMonitoringData) -if __name__ == "__main__": - testSuite = unittest.defaultTestLoader.loadTestsFromTestCase(MonitoringTestCase) - testSuite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MonitoringReporterAdd)) - testResult = unittest.TextTestRunner(verbosity=2).run(testSuite) - sys.exit(not testResult.wasSuccessful()) +def test_addPilotRecords(): + for record in pilotMonitoringData: + pilotMonitoringReporter.addRecord(record) + result = pilotMonitoringReporter.commit() + assert result["OK"] + assert result["Value"] == len(pilotMonitoringData)