Merge pull request DIRACGrid#7498 from fstagni/removedGridEnv

[9.0] Removed grid env

dirac.cfg

@@ -624,9 +624,6 @@ Resources
     CEDefaults
     {
 
-      # Default environment file sourced before calling rid commands, without extension '.sh'
-      GridEnv = /opt/dirac/gridenv
-
       # Will be added to the pilot configuration as /LocalSite/SharedArea
       SharedArea = /cvmfs/lhcb.cern.ch/lib
 

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Agents/PilotStatusAgent/index.rst

@@ -8,9 +8,6 @@ Special attributes for this agent are:
 +--------------------------+--------------------------------------------+-----------------------------------+
 | **Name**                 | **Description**                            | **Example**                       |
 +--------------------------+--------------------------------------------+-----------------------------------+
-| *GridEnv*                | Path where is located the file to          | GridEnv = /usr/profile.d/grid-env |
-|                          | load Grid Environment Variables            |                                   |
-+--------------------------+--------------------------------------------+-----------------------------------+
 | *PilotAccountingEnabled* | Boolean type attribute than allows to      | PilotAccountingEnabled = Yes      |
 |                          | specify if accounting is enabled           |                                   |
 +--------------------------+--------------------------------------------+-----------------------------------+

docs/source/AdministratorGuide/HowTo/multiVO.rst

@@ -206,15 +206,6 @@ Resources/StorageElements/ProductionSandboxSE
      }
    }
 
-WorkloadManagement - PilotStatusAgent
--------------------------------------
-
-Option value could be different, it depends on UI
-installed on server
-::
-
-   Systems/WorkloadManagement/<setup>/Agents/PilotStatusAgent/GridEnv = /etc/profile.d/grid-env
-
 
 
 DONE

src/DIRAC/Core/Utilities/Grid.py

@@ -1,47 +1,9 @@
 """
 The Grid module contains several utilities for grid operations
 """
-import os
 
-from DIRAC.Core.Utilities.Os import sourceEnv
-from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
-from DIRAC.Core.Security.ProxyInfo import getProxyInfo
-from DIRAC.ConfigurationSystem.Client.Helpers import Local
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
-from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall
-
-
-def executeGridCommand(cmd, gridEnvScript=None, gridEnvDict=None):
-    """
-    Execute cmd tuple after sourcing GridEnv
-    """
-    currentEnv = dict(os.environ)
-
-    if not gridEnvScript:
-        # if not passed as argument, use default from CS Helpers
-        gridEnvScript = Local.gridEnv()
-
-    if gridEnvScript:
-        command = gridEnvScript.split()
-        ret = sourceEnv(10, command)
-        if not ret["OK"]:
-            return S_ERROR(f"Failed sourcing GridEnv: {ret['Message']}")
-        gridEnv = ret["outputEnv"]
-        #
-        # Preserve some current settings if they are there
-        #
-        if "X509_VOMS_DIR" in currentEnv:
-            gridEnv["X509_VOMS_DIR"] = currentEnv["X509_VOMS_DIR"]
-        if "X509_CERT_DIR" in currentEnv:
-            gridEnv["X509_CERT_DIR"] = currentEnv["X509_CERT_DIR"]
-    else:
-        gridEnv = currentEnv
-
-    if gridEnvDict:
-        gridEnv.update(gridEnvDict)
-
-    result = systemCall(120, cmd, env=gridEnv)
-    return result
+from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
+from DIRAC.Core.Utilities.Subprocess import shellCall
 
 
 def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString="Glue"):

src/DIRAC/Resources/Computing/ARCComputingElement.py

@@ -51,16 +51,16 @@
 import uuid
 
 import arc  # Has to work if this module is called #pylint: disable=import-error
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities.Subprocess import shellCall
-from DIRAC.Core.Utilities.List import breakListIntoChunks
+
+from DIRAC import S_ERROR, S_OK
 from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.Core.Utilities.Decorators import deprecated
+from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.Core.Utilities.Subprocess import shellCall
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
 from DIRAC.Resources.Computing.PilotBundle import writeScript
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 
-
 MANDATORY_PARAMETERS = ["Queue"]  # Mandatory for ARC CEs in GLUE2?
 # See https://www.nordugrid.org/arc/arc6/tech/rest/rest.html#rest-interface-job-states
 # We let "Deleted, Hold, Undefined" for the moment as we are not sure whether they are still used
@@ -127,15 +127,13 @@ def __init__(self, ceUniqueID):
         self.pilotProxy = ""
         self.queue = ""
         self.arcQueue = ""
-        self.gridEnv = ""
         self.ceHost = self.ceName
         self.endpointType = "Gridftp"
         self.usercfg = arc.common.UserConfig()
         self.preamble = ""
 
         # set the timeout to the default 20 seconds in case the UserConfig constructor did not
         self.usercfg.Timeout(20)  # pylint: disable=pointless-statement
-        self.gridEnv = ""
 
         # Used in getJobStatus
         self.mapStates = STATES_MAP
@@ -273,7 +271,6 @@ def _reset(self):
         self.arcQueue = self.queue.split("-", 2)[2]
 
         self.ceHost = self.ceParameters.get("Host", self.ceHost)
-        self.gridEnv = self.ceParameters.get("GridEnv", self.gridEnv)
 
         # extra XRSL data (should respect the XRSL format)
         self.xrslExtraString = self.ceParameters.get("XRSLExtraString", self.xrslExtraString)

src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py

@@ -49,20 +49,20 @@
 import os
 import subprocess
 import tempfile
-import threading
 import textwrap
+import threading
 import uuid
 
 from DIRAC import S_ERROR, S_OK, gConfig
-from DIRAC.Core.Utilities.File import makeGuid, mkDir
-from DIRAC.Core.Utilities.Grid import executeGridCommand
+from DIRAC.Core.Security.Locations import getCAsLocation
+from DIRAC.Core.Utilities.File import mkDir
 from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.Core.Utilities.Subprocess import systemCall
+from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import writeToTokenFile
+from DIRAC.Resources.Computing.BatchSystems.Condor import HOLD_REASON_SUBCODE, parseCondorStatus, subTemplate
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
-from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import writeToTokenFile
-from DIRAC.Core.Security.Locations import getCAsLocation
-from DIRAC.Resources.Computing.BatchSystems.Condor import HOLD_REASON_SUBCODE, subTemplate, parseCondorStatus
 
 MANDATORY_PARAMETERS = ["Queue"]
 DEFAULT_WORKINGDIRECTORY = "/opt/dirac/pro/runit/WorkloadManagement/SiteDirectorHT"
@@ -91,7 +91,6 @@ def __init__(self, ceUniqueID):
         self.pilotProxy = ""
         self.queue = ""
         self.outputURL = "gsiftp://localhost"
-        self.gridEnv = ""
         self.proxyRenewal = 0
         self.daysToKeepLogs = DEFAULT_DAYSTOKEEPLOGS
         self.daysToKeepRemoteLogs = DEFAULT_DAYSTOKEEPREMOTELOGS
@@ -203,7 +202,6 @@ def __writeSub(self, executable, location, processors, pilotStamps, tokenFile=No
     def _reset(self):
         self.queue = self.ceParameters["Queue"]
         self.outputURL = self.ceParameters.get("OutputURL", "gsiftp://localhost")
-        self.gridEnv = self.ceParameters.get("GridEnv")
         self.daysToKeepLogs = self.ceParameters.get("DaysToKeepLogs", DEFAULT_DAYSTOKEEPLOGS)
         self.extraSubmitString = str(self.ceParameters.get("ExtraSubmitString", "").encode().decode("unicode_escape"))
         self.daysToKeepRemoteLogs = self.ceParameters.get("DaysToKeepRemoteLogs", DEFAULT_DAYSTOKEEPREMOTELOGS)
@@ -225,7 +223,7 @@ def _executeCondorCommand(self, cmd, keepTokenFile=False):
 
         :param list cmd: list of the condor command elements
         :param bool keepTokenFile: flag to reuse or not the previously created token file
-        :return: S_OK/S_ERROR - the stdout parameter of the executeGridCommand() call
+        :return: S_OK/S_ERROR - the stdout parameter of the systemCall() call
         """
         if not self.token and not self.proxy:
             return S_ERROR(f"Cannot execute the command, token and proxy not found: {cmd}")
@@ -255,11 +253,9 @@ def _executeCondorCommand(self, cmd, keepTokenFile=False):
                 htcEnv["_CONDOR_AUTH_SSL_CLIENT_CADIR"] = cas
 
         # Execute the command
-        result = executeGridCommand(
-            cmd,
-            gridEnvScript=self.gridEnv,
-            gridEnvDict=htcEnv,
-        )
+        currentEnv = dict(os.environ)
+        currentEnv.update(htcEnv)
+        result = systemCall(120, cmd, env=currentEnv)
         if not result["OK"]:
             self.tokenFile = None
             self.log.error("Command", f"{cmd} failed with: {result['Message']}")

src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py

@@ -3,11 +3,12 @@
 tests for HTCondorCEComputingElement module
 """
 import uuid
+
 import pytest
 
+from DIRAC import S_OK
 from DIRAC.Resources.Computing import HTCondorCEComputingElement as HTCE
 from DIRAC.Resources.Computing.BatchSystems import Condor
-from DIRAC import S_OK
 
 MODNAME = "DIRAC.Resources.Computing.HTCondorCEComputingElement"
 
@@ -27,7 +28,7 @@
 
 @pytest.fixture
 def setUp():
-    return {"Queue": "espresso", "GridEnv": "/dev/null"}
+    return {"Queue": "espresso"}
 
 
 def test_parseCondorStatus():
@@ -72,7 +73,7 @@ def test_parseCondorStatus():
 def test_getJobStatus(mocker):
     """Test HTCondorCE getJobStatus"""
     mocker.patch(
-        MODNAME + ".executeGridCommand",
+        MODNAME + ".systemCall",
         side_effect=[
             S_OK((0, "\n".join(STATUS_LINES), "")),
             S_OK((0, "\n".join(HISTORY_LINES), "")),
@@ -186,7 +187,7 @@ def test_submitJob(setUp, mocker, localSchedd, expected):
     ceName = "condorce.cern.ch"
     htce.ceName = ceName
 
-    execMock = mocker.patch(MODNAME + ".executeGridCommand", return_value=S_OK((0, "123.0 - 123.0", "")))
+    execMock = mocker.patch(MODNAME + ".systemCall", return_value=S_OK((0, "123.0 - 123.0", "")))
     mocker.patch(MODNAME + ".HTCondorCEComputingElement._prepareProxy", return_value=S_OK())
     mocker.patch(
         MODNAME + ".HTCondorCEComputingElement._HTCondorCEComputingElement__writeSub", return_value="dirac_pilot"
@@ -196,7 +197,7 @@ def test_submitJob(setUp, mocker, localSchedd, expected):
     result = htce.submitJob("pilot", "proxy", 1)
 
     assert result["OK"] is True
-    assert " ".join(execMock.call_args_list[0][0][0]) == expected
+    assert " ".join(execMock.call_args_list[0][0][1]) == expected
 
 
 @pytest.mark.parametrize(
@@ -220,11 +221,11 @@ def test_killJob(setUp, mocker, jobIDList, jobID, ret, success, local):
     htce.ceParameters = ceParameters
     htce._reset()
 
-    execMock = mocker.patch(MODNAME + ".executeGridCommand", return_value=S_OK((ret, "", "")))
+    execMock = mocker.patch(MODNAME + ".systemCall", return_value=S_OK((ret, "", "")))
     mocker.patch(MODNAME + ".HTCondorCEComputingElement._prepareProxy", return_value=S_OK())
 
     ret = htce.killJob(jobIDList=jobIDList)
     assert ret["OK"] == success
     if jobID:
         expected = f"condor_rm {htce.remoteScheddOptions.strip()} {jobID}"
-        assert " ".join(execMock.call_args_list[0][0][0]) == expected
+        assert " ".join(execMock.call_args_list[0][0][1]) == expected

src/DIRAC/TransformationSystem/test/Test_JobInfo.py

@@ -76,7 +76,6 @@ def setUp(self):
             ],
             "StdOutput": "std.out",
             "JobType": "MCReconstruction_Overlay",
-            "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
             "TransformationID": 6326,
             "StdError": "std.err",
             "IS_PROD": "True",
@@ -143,7 +142,6 @@ def setUp(self):
             ],
             "StdOutput": "std.out",
             "JobType": "MCReconstruction_Overlay",
-            "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
             "TransformationID": 6326,
             "StdError": "std.err",
             "IS_PROD": "True",
@@ -214,7 +212,6 @@ def setUp(self):
             ],
             "StdOutput": "std.out",
             "JobType": "MCSimulation",
-            "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
             "TransformationID": 6301,
             "StdError": "std.err",
             "IS_PROD": "True",
@@ -271,7 +268,6 @@ def setUp(self):
             ],
             "StdOutput": "std.out",
             "JobType": "MCGeneration",
-            "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example",
             "TransformationID": 6498,
             "StdError": "std.err",
             "IS_PROD": "True",

src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py

@@ -45,7 +45,6 @@ def __init__(self, *args, **kwargs):
     def initialize(self):
         """Sets defaults"""
 
-        self.am_setOption("GridEnv", "")
         self.pilotDB = PilotAgentsDB()
         self.diracadmin = DiracAdmin()
         self.jobDB = JobDB()
@@ -60,14 +59,6 @@ def execute(self):
         """The PilotAgent execution method."""
 
         self.pilotStalledDays = self.am_getOption("PilotStalledDays", 3)
-        self.gridEnv = self.am_getOption("GridEnv")
-        if not self.gridEnv:
-            # No specific option found, try a general one
-            setup = gConfig.getValue("/DIRAC/Setup", "")
-            if setup:
-                instance = gConfig.getValue(f"/DIRAC/Setups/{setup}/WorkloadManagement", "")
-                if instance:
-                    self.gridEnv = gConfig.getValue(f"/Systems/WorkloadManagement/{instance}/GridEnv", "")
 
         result = self.pilotDB._getConnection()
         if not result["OK"]:

src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py

@@ -4,12 +4,11 @@
 import shutil
 from tempfile import mkdtemp
 
-from DIRAC import S_ERROR, S_OK, gConfig, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Registry import (
     getDNForUsername,
     getGroupOption,
-    getVOForGroup,
 )
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueue
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
@@ -22,18 +21,6 @@
 outputSandboxFiles = ["StdOut", "StdErr"]
 
 COMMAND_TIMEOUT = 60
-###########################################################################
-
-
-def getGridEnv():
-    gridEnv = ""
-    setup = gConfig.getValue("/DIRAC/Setup", "")
-    if setup:
-        instance = gConfig.getValue(f"/DIRAC/Setups/{setup}/WorkloadManagement", "")
-        if instance:
-            gridEnv = gConfig.getValue(f"/Systems/WorkloadManagement/{instance}/GridEnv", "")
-
-    return gridEnv
 
 
 def getPilotCE(pilotDict):
@@ -43,8 +30,6 @@ def getPilotCE(pilotDict):
     if not result["OK"]:
         return result
     queueDict = result["Value"]
-    gridEnv = getGridEnv()
-    queueDict["GridEnv"] = gridEnv
     queueDict["WorkingDirectory"] = mkdtemp()
     result = ceFactory.getCE(pilotDict["GridType"], pilotDict["DestinationSite"], queueDict)
     if not result["OK"]: