diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py index 4150b2a4c89..c01cb673dc1 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py @@ -98,7 +98,6 @@ def main(): DIRAC.exit() STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] - ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded", "Error"] statusAllowedDict = {} for statusType in STATUS_TYPES: @@ -123,26 +122,14 @@ def main(): # InActive is used on the CS model, Banned is the equivalent in RSS for statusType in STATUS_TYPES: if statusFlagDict[statusType]: - if seOptions.get(statusType) == "Active": - gLogger.notice(f"{statusType} status of {se} is already Active") - continue if statusType in seOptions: - if not seOptions[statusType] in ALLOWED_STATUSES: - gLogger.notice( - "%s option for %s is %s, instead of %s" - % (statusType, se, seOptions["ReadAccess"], ALLOWED_STATUSES) - ) - gLogger.notice("Try specifying the command switches") + resR = resourceStatus.setElementStatus(se, "StorageElement", statusType, "Active", reason, userName) + if not resR["OK"]: + gLogger.fatal(f"Failed to update {se} {statusType} to Active, exit -", resR["Message"]) + DIRAC.exit(-1) else: - resR = resourceStatus.setElementStatus( - se, "StorageElement", statusType, "Active", reason, userName - ) - if not resR["OK"]: - gLogger.fatal(f"Failed to update {se} {statusType} to Active, exit -", resR["Message"]) - DIRAC.exit(-1) - else: - gLogger.notice(f"Successfully updated {se} {statusType} to Active") - statusAllowedDict[statusType].append(se) + gLogger.notice(f"Successfully updated {se} {statusType} to Active") + statusAllowedDict[statusType].append(se) totalAllowed = 0 totalAllowedSEs = [] diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py index 91553dc2a06..3e323d7e63a 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py @@ -113,83 +113,43 @@ def main(): # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: - if seOptions["ReadAccess"] == "Banned": - gLogger.notice("Read access already banned", se) - resR["OK"] = True - elif not seOptions["ReadAccess"] in ["Active", "Degraded", "Probing", "Error"]: - gLogger.notice( - "Read option for %s is %s, instead of %s" - % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing", "Error"]) - ) - gLogger.notice("Try specifying the command switches") + resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) + # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) + if not resR["OK"]: + gLogger.error(f"Failed to update {se} read access to Banned") else: - resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) - # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) - if not resR["OK"]: - gLogger.error(f"Failed to update {se} read access to Banned") - else: - gLogger.notice(f"Successfully updated {se} read access to Banned") - readBanned.append(se) + gLogger.notice(f"Successfully updated {se} read access to Banned") + readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: - if seOptions["WriteAccess"] == "Banned": - gLogger.notice("Write access already banned", se) - resW["OK"] = True - elif not seOptions["WriteAccess"] in ["Active", "Degraded", "Probing"]: - gLogger.notice( - "Write option for %s is %s, instead of %s" - % (se, seOptions["WriteAccess"], ["Active", "Degraded", "Probing"]) - ) - gLogger.notice("Try specifying the command switches") + resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) + # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) + if not resW["OK"]: + gLogger.error(f"Failed to update {se} write access to Banned") else: - resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) - # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) - if not resW["OK"]: - gLogger.error(f"Failed to update {se} write access to Banned") - else: - gLogger.notice(f"Successfully updated {se} write access to Banned") - writeBanned.append(se) + gLogger.notice(f"Successfully updated {se} write access to Banned") + writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: - if seOptions["CheckAccess"] == "Banned": - gLogger.notice("Check access already banned", se) - resC["OK"] = True - elif not seOptions["CheckAccess"] in ["Active", "Degraded", "Probing"]: - gLogger.notice( - "Check option for %s is %s, instead of %s" - % (se, seOptions["CheckAccess"], ["Active", "Degraded", "Probing"]) - ) - gLogger.notice("Try specifying the command switches") + resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC["OK"]: + gLogger.error(f"Failed to update {se} check access to Banned") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC["OK"]: - gLogger.error(f"Failed to update {se} check access to Banned") - else: - gLogger.notice(f"Successfully updated {se} check access to Banned") - checkBanned.append(se) + gLogger.notice(f"Successfully updated {se} check access to Banned") + checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: - if seOptions["RemoveAccess"] == "Banned": - gLogger.notice("Remove access already banned", se) - resC["OK"] = True - elif not seOptions["RemoveAccess"] in ["Active", "Degraded", "Probing"]: - gLogger.notice( - "Remove option for %s is %s, instead of %s" - % (se, seOptions["RemoveAccess"], ["Active", "Degraded", "Probing"]) - ) - gLogger.notice("Try specifying the command switches") + resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC["OK"]: + gLogger.error(f"Failed to update {se} remove access to Banned") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC["OK"]: - gLogger.error(f"Failed to update {se} remove access to Banned") - else: - gLogger.notice(f"Successfully updated {se} remove access to Banned") - removeBanned.append(se) + gLogger.notice(f"Successfully updated {se} remove access to Banned") + removeBanned.append(se) if not (resR["OK"] or resW["OK"] or resC["OK"]): DIRAC.exit(-1) diff --git a/src/DIRAC/Interfaces/API/DiracAdmin.py b/src/DIRAC/Interfaces/API/DiracAdmin.py index e5dfa56f555..ef61c4ee187 100755 --- a/src/DIRAC/Interfaces/API/DiracAdmin.py +++ b/src/DIRAC/Interfaces/API/DiracAdmin.py @@ -107,7 +107,7 @@ def getSiteMask(self, printOutput=False, status="Active"): ############################################################################# def getBannedSites(self, printOutput=False): - """Retrieve current list of banned and probing sites. + """Retrieve current list of banned sites. Example usage: @@ -122,16 +122,12 @@ def getBannedSites(self, printOutput=False): if not bannedSites["OK"]: return bannedSites - probingSites = self.sitestatus.getSites(siteState="Probing") - if not probingSites["OK"]: - return probingSites - - mergedList = sorted(bannedSites["Value"] + probingSites["Value"]) + bannedList = sorted(bannedSites["Value"]) if printOutput: - gLogger.notice("\n".join(mergedList)) + gLogger.notice("\n".join(bannedList)) - return S_OK(mergedList) + return S_OK(bannedList) ############################################################################# def getSiteSection(self, site, printOutput=False): @@ -170,14 +166,6 @@ def allowSite(self, site, comment, printOutput=False, days=1): if not (result := self._checkSiteIsValid(site))["OK"]: return result - if not (result := self.getSiteMask(status="Active"))["OK"]: - return result - siteMask = result["Value"] - if site in siteMask: - if printOutput: - gLogger.notice(f"Site {site} is already Active") - return S_OK(f"Site {site} is already Active") - tokenLifetime = int(days) if tokenLifetime <= 0: tokenExpiration = datetime.max @@ -246,14 +234,6 @@ def banSite(self, site, comment, printOutput=False, days=1): """ if not (result := self._checkSiteIsValid(site))["OK"]: return result - mask = self.getSiteMask(status="Banned") - if not mask["OK"]: - return mask - siteMask = mask["Value"] - if site in siteMask: - if printOutput: - gLogger.notice(f"Site {site} is already Banned") - return S_OK(f"Site {site} is already Banned") tokenLifetime = int(days) if tokenLifetime <= 0: diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py index c2b6424662c..09fb3cebaa5 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py @@ -17,6 +17,7 @@ from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import getPoliciesThatApply from DIRAC.ResourceStatusSystem.Utilities.RSSCacheNoThread import RSSCache from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration +from DIRAC.ResourceStatusSystem.PolicySystem.StateMachine import resolveElementStatus class ResourceStatus(metaclass=DIRACSingleton): @@ -66,13 +67,13 @@ def getElementStatus(self, elementName, elementType, statusType=None, default=No >>> getElementStatus('SE1', 'StorageElement', ['ReadAccess', 'WriteAccess']) S_OK( { 'SE1': { 'ReadAccess': 'Banned' , 'WriteAccess': 'Active'} } } ) >>> getElementStatus('SE1', 'StorageElement') - S_OK( { 'SE1': { 'ReadAccess': 'Probing' , + S_OK( { 'SE1': { 'ReadAccess': 'Banned' , 'WriteAccess': 'Active', - 'CheckAccess': 'Degraded', + 'CheckAccess': 'Active', 'RemoveAccess': 'Banned'} } } ) >>> getElementStatus(['CE1', 'CE2'], 'ComputingElement') S_OK( {'CE1': {'all': 'Active'}, - 'CE2': {'all': 'Probing'}}} + 'CE2': {'all': 'Banned'}}} """ allowedParameters = ["StorageElement", "ComputingElement", "FTS"] @@ -257,9 +258,7 @@ def getCacheDictFromRawData(rawList): :return: dict of the form { ( elementName, elementType, statusType, vO ) : status, ... } """ - res = {} for entry in rawList: - res.update({(entry[0], entry[1], entry[2], entry[4]): entry[3]}) - + res[(entry[0], entry[1], entry[2], entry[4])] = resolveElementStatus(entry[3]) return res diff --git a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py index cd5afd35697..c24541f477a 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py @@ -18,6 +18,7 @@ from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient from DIRAC.ResourceStatusSystem.Utilities.RSSCacheNoThread import RSSCache from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration +from DIRAC.ResourceStatusSystem.PolicySystem.StateMachine import resolveElementStatus class SiteStatus(metaclass=DIRACSingleton): @@ -125,7 +126,7 @@ def __getRSSSiteStatus(self, siteName=None): def getUsableSites(self, siteNames=None): """ Returns all sites that are usable if their - statusType is either Active or Degraded; in a list. + statusType is either Active; in a list. examples >>> siteStatus.getUsableSites( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] ) @@ -147,7 +148,7 @@ def getUsableSites(self, siteNames=None): return siteStatusDictRes if not siteStatusDictRes["Value"]: return S_OK([]) - return S_OK([x[0] for x in siteStatusDictRes["Value"].items() if x[1] in ["Active", "Degraded"]]) + return S_OK([x[0] for x in siteStatusDictRes["Value"].items() if x[1] == "Active"]) def getSites(self, siteState="Active"): """ @@ -188,7 +189,7 @@ def getSites(self, siteState="Active"): else: # fix case sensitive string siteState = siteState.capitalize() - allowedStateList = ["Active", "Banned", "Degraded", "Probing", "Error", "Unknown"] + allowedStateList = ["Active", "Banned"] if siteState not in allowedStateList: return S_ERROR(errno.EINVAL, "Not a valid status, parameter rejected") @@ -275,9 +276,7 @@ def getCacheDictFromRawData(rawList): :return: dict of the form { ( elementName ) : status, ... } """ - res = {} for entry in rawList: - res.update({(entry[0]): entry[1]}) - + res[entry[0]] = resolveElementStatus(entry[1]) return res diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/StateMachine.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/StateMachine.py index 3e6b8b7095a..7f7292e039a 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/StateMachine.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/StateMachine.py @@ -86,3 +86,21 @@ def getLevelOfPolicyState(self, policyResult): """ return self.getLevelOfState(policyResult["Status"]) + + +def resolveElementStatus(status): + """ + Maps an internal RSS status onto the simplified 'Active' / 'Banned' status + exposed to clients. + + examples: + >>> resolveElementStatus( 'Degraded' ) + 'Active' + >>> resolveElementStatus( 'Unknown' ) + 'Banned' + + :param str status: the internal RSS status + :return: str, 'Active' or 'Banned' + """ + ALLOWED = {"Active", "Degraded"} + return "Active" if status in ALLOWED else "Banned" diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 902f0c33e9b..0e91ddfd843 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -547,21 +547,13 @@ def status(self): # If nothing is defined in the CS Access is allowed # If something is defined, then it must be set to Active - retDict["Read"] = not ( - "ReadAccess" in self.options and self.options["ReadAccess"] not in ("Active", "Degraded") - ) - retDict["Write"] = not ( - "WriteAccess" in self.options and self.options["WriteAccess"] not in ("Active", "Degraded") - ) - retDict["Remove"] = not ( - "RemoveAccess" in self.options and self.options["RemoveAccess"] not in ("Active", "Degraded") - ) + retDict["Read"] = not ("ReadAccess" in self.options and self.options["ReadAccess"] != "Active") + retDict["Write"] = not ("WriteAccess" in self.options and self.options["WriteAccess"] != "Active") + retDict["Remove"] = not ("RemoveAccess" in self.options and self.options["RemoveAccess"] != "Active") if retDict["Read"]: retDict["Check"] = True else: - retDict["Check"] = not ( - "CheckAccess" in self.options and self.options["CheckAccess"] not in ("Active", "Degraded") - ) + retDict["Check"] = not ("CheckAccess" in self.options and self.options["CheckAccess"] != "Active") diskSE = True tapeSE = False if "SEType" in self.options: diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index ef4a14426f4..715d2d489a0 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -222,7 +222,7 @@ def _buildQueueDict( self.log.error("Can not get the status of computing elements: ", result["Message"]) return result # Try to get CEs which have been probed and those unprobed (vO='all'). - ceMaskList = [ceName for ceName in result["Value"] if result["Value"][ceName]["all"] in ("Active", "Degraded")] + ceMaskList = [ceName for ceName in result["Value"] if result["Value"][ceName]["all"] == "Active"] # Filter the unusable queues for queueName in list(self.queueDict.keys()): diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index d1b1f08c096..d524abcd87b 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -1094,10 +1094,6 @@ def getSiteSummaryWeb(self, selectDict, sortList, startItem, maxItems): # Get the site mask status siteMask = {} - resultMask = self.siteClient.getSites("All") - if resultMask["OK"]: - for site in resultMask["Value"]: - siteMask[site] = "NoMask" resultMask = self.siteClient.getSites("Active") if resultMask["OK"]: for site in resultMask["Value"]: diff --git a/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py b/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py index a7eace12241..7633cc81a98 100644 --- a/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py +++ b/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py @@ -15,6 +15,7 @@ from DIRAC import gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient +from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus gLogger.setLevel("DEBUG") @@ -26,6 +27,12 @@ def fixtureResourceStatusClient(): yield ResourceStatusClient() +@pytest.fixture(name="rsClient") +def fixtureResourceStatus(): + resourceStatus = ResourceStatus() + yield resourceStatus + + def test_addAndRemove(rssClient: ResourceStatusClient): # clean up rssClient.deleteStatusElement("Site", "Status", "TestSite1234") @@ -422,3 +429,76 @@ def test_addIfNotThereStatusElement(rssClient: ResourceStatusClient): assert res["OK"] is True, res["Message"] # check if the returned value is empty assert not res["Value"] + + +def test_getElementStatus(rssClient: ResourceStatusClient, rsClient): + # make sure that the test resoureces are not presented in the db + rssClient.deleteStatusElement("Resource", "Status", "testActiveResource") + rssClient.deleteStatusElement("Resource", "Status", "testBannedResource") + rssClient.deleteStatusElement("Resource", "Status", "testResource") + + res = rssClient.insertStatusElement( + "Resource", + "Status", + "testActiveResource", + "all", + "Degraded", + "ComputingElement", + "reason", + Datetime, + Datetime, + "tokenOwner", + Datetime, + ) + + assert res["OK"] is True, res["Message"] + rsClient.rssCache.refreshCache() + + res = rsClient.getElementStatus("testActiveResource", "ComputingElement") + + assert res["OK"] is True, res["Message"] + assert res["Value"]["testActiveResource"]["all"] == "Active" + + res = rssClient.insertStatusElement( + "Resource", + "Status", + "testBannedResource", + "all", + "Probing", + "ComputingElement", + "reason", + Datetime, + Datetime, + "tokenOwner", + Datetime, + ) + + assert res["OK"] is True, res["Message"] + rsClient.rssCache.refreshCache() + + res = rsClient.getElementStatus("testBannedResource", "ComputingElement") + assert res["OK"] is True, res["Message"] + assert res["Value"]["testBannedResource"]["all"] == "Banned" + + res = rssClient.insertStatusElement( + "Resource", + "Status", + "testResource", + "all", + "Active", + "ComputingElement", + "reason", + Datetime, + Datetime, + "tokenOwner", + Datetime, + ) + assert res["OK"] is True, res["Message"] + rsClient.rssCache.refreshCache() + + res = rsClient.setElementStatus("testResource", "ComputingElement", "all", "Error") + assert res["OK"] is True, res["Message"] + rsClient.rssCache.refreshCache() + res = rsClient.getElementStatus("testResource", "ComputingElement") + assert res["OK"] is True, res["Message"] + assert res["Value"]["testResource"]["all"] == "Banned" diff --git a/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py b/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py index 4fa16c06024..bd8f24ba1ca 100644 --- a/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py +++ b/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py @@ -132,7 +132,7 @@ def test_addAndRemove_complicatedTest(stClient): result = stClient.getSites() assert result["OK"] is True, result["Message"] - inRSS = "testBanned.test.test" in result["Value"] + inRSS = "testActive.test.test" in result["Value"] # TEST getSites # ............................................................................... @@ -148,9 +148,10 @@ def test_addAndRemove_complicatedTest(stClient): # setting a status if inRSS: - result = stClient.setSiteStatus("testBanned.test.test", "Probing") + result = stClient.setSiteStatus("testActive.test.test", "Probing") assert result["OK"] is True, result["Message"] stClient.rssCache.refreshCache() - result = stClient.getSites("Probing") + result = stClient.getSites("Banned") assert result["OK"] is True, result["Message"] + assert "testActive.test.test" in result["Value"]