Merge pull request #7625 from fstagni/90_downloadTest

test: added a workflow test with input data

src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py

@@ -1,13 +1,11 @@
-""" The Input Data By Protocol module wraps around the Replica Management
+""" The Input Data By Protocol module wraps around the StorageElement
     components to provide access to datasets by available site protocols as
     defined in the CS for the VO.
 """
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 
-COMPONENT_NAME = "InputDataByProtocol"
-
 
 class InputDataByProtocol:
     #############################################################################

tests/.dirac-ci-config.yaml

@@ -1,6 +1,6 @@
 config:
   DIRAC_CI_SETUP_SCRIPT: DIRAC/tests/Jenkins/dirac_ci.sh
-  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 1 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug
+  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug
   PILOT_JSON: "{
         \"timestamp\": \"2023-02-13T14:34:26.725499\",
         \"CEs\": {

tests/CI/install_client.sh

@@ -57,13 +57,30 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n"
 
 installDIRAC
 
-#-------------------------------------------------------------------------------#
-echo -e "*** $(date -u) **** Submit a job ****\n"
-
 echo -e "*** $(date -u)  Getting a non privileged user\n" |& tee -a clientTestOutputs.txt
 dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt
 
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** Submit a job ****\n"
+
 echo -e '[\n    Arguments = "Hello World";\n    Executable = "echo";\n    Site = "DIRAC.Jenkins.ch";' > test.jdl
 echo "    JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl
 echo "]" >> test.jdl
-dirac-wms-job-submit test.jdl
+dirac-wms-job-submit test.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt
+
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** add a file ****\n"
+
+echo "I like pizza!" > test_lfn.txt
+ls -l
+echo $?
+dirac-dms-add-file LFN:/vo/test_lfn.txt test_lfn.txt S3-DIRECT
+echo $?
+
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** Submit a job with an input ****\n"
+
+echo -e '[\n    Arguments = "Hello World";\n    Executable = "echo";\n    Site = "DIRAC.Jenkins.ch";\n    InputData = "/vo/test_lfn.txt";' > test_dl.jdl
+echo "    JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl
+echo "]" >> test_dl.jdl
+dirac-wms-job-submit test_dl.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt

tests/Integration/WorkloadManagementSystem/Test_JobDB.py

@@ -154,6 +154,7 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB):
         CPUTime = 86400;
         Executable = "dirac-jobexec";
         InputData = "";
+        InputDataModule = "DIRAC.WorkloadManagementSystem.Client.DownloadInputData";
         InputSandbox =
             {{
                 "../../Integration/WorkloadManagementSystem/exe-script.py",

tests/Jenkins/dirac-cfg-update-server.py

@@ -127,6 +127,7 @@
 #           DIRAC.Jenkins.ch
 #           {
 #             SE = SE-1
+#             SE += S3-DIRECT
 #             CEs
 #             {
 #               jenkins.cern.ch
@@ -163,7 +164,7 @@
         print(res["Message"])
         sys.exit(1)
 
-csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1")
+csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1,S3-DIRECT")
 csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/CEType", "Test")
 csAPI.setOption(
     "Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/Queues/jenkins-queue_not_important/maxCPUTime", "200000"
@@ -435,6 +436,31 @@
 csAPI.setOption("Operations/Defaults/Services/Catalogs/CatalogList", "FileCatalog, TSCatalog, MultiVOFileCatalog")
 
 
+# Adding InputDataPolicy section of Operations
+# Operations
+# {
+#   Defaults
+#   {
+#     InputDataPolicy
+#     {
+#      Download = DIRAC.WorkloadManagementSystem.Client.DownloadInputData
+#      InputDataModule = DIRAC.WorkloadManagementSystem.Client.DownloadInputData
+#     }
+#   }
+# }
+
+res = csAPI.createSection("Operations/Defaults/InputDataPolicy")
+if not res["OK"]:
+    print(res["Message"])
+    sys.exit(1)
+csAPI.setOption(
+    "Operations/Defaults/InputDataPolicy/Download", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"
+)
+csAPI.setOption(
+    "Operations/Defaults/InputDataPolicy/InputDataModule", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"
+)
+
+
 # Adding DataManagement section of Operations
 # Operations
 # {

tests/Workflow/Integration/Test_UserJobs.py

@@ -3,22 +3,21 @@
     Can be automatized.
 """
 # pylint: disable=protected-access, wrong-import-position, invalid-name, missing-docstring
+import multiprocessing
 import os
 import sys
 import unittest
-import multiprocessing
 
 import DIRAC
 
 DIRAC.initialize(extra_config_files=["pilot.cfg"])  # Initialize configuration
 
 from DIRAC import gLogger, rootPath
+from DIRAC.Interfaces.API.Dirac import Dirac
+from DIRAC.Interfaces.API.Job import Job
 from DIRAC.tests.Utilities.IntegrationTest import IntegrationTest
 from DIRAC.tests.Utilities.utils import find_all
 
-from DIRAC.Interfaces.API.Job import Job
-from DIRAC.Interfaces.API.Dirac import Dirac
-
 
 class UserJobTestCase(IntegrationTest):
     """Base class for the UserJob test cases"""
@@ -154,6 +153,27 @@ def test_execute(self):
         self.assertTrue(res["OK"])
 
 
+class DownloadSuccess(UserJobTestCase):
+    def test_execute(self):
+        """Testing a job that downloads inputs"""
+
+        job = Job()
+
+        job.setName("dl-test")
+        job.setExecutable("/bin/ls", "-l")
+        job.setLogLevel("DEBUG")
+        try:
+            # This is the standard location in Jenkins
+            job.setInputSandbox(find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
+        except (IndexError, KeyError):
+            job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
+        job.setConfigArgs("pilot.cfg")
+        job.setInputData("/vo/test_lfn.txt")
+        job.setInputDataPolicy("Download")
+        res = job.runLocal(self.d)
+        self.assertTrue(res["OK"])
+
+
 class MPSuccess(UserJobTestCase):
     def test_fixed(self):
         """this tests executes a job that requires exactly 4 processors"""
@@ -245,6 +265,7 @@ def test_min1(self):
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldPlusSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(LSSuccess))
+    suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DownloadSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccessMinMax))
     testResult = unittest.TextTestRunner(verbosity=2).run(suite)