diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index dcccf39dc73..be2be9b1494 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -134,26 +134,6 @@ jobs: env: REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} - pylint-py27: - name: Pylint for Python 2.7 in Pilot files - runs-on: ubuntu-latest - if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' - steps: - - uses: actions/checkout@v6 - - name: Fail-fast for outdated pipelines - run: .github/workflows/fail-fast.sh - - name: prepare environment - run: | - conda config --set add_pip_as_python_dependency false - conda create -c conda-forge -n test-env python=2.7 pylint=1.9.2 astroid=1.6.5 - - name: run pilot wrapper test - run: | - eval "$(conda shell.bash hook)" && conda activate test-env - pylint -j 0 -E \ - tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \ - src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \ - src/DIRAC/Resources/Computing/BatchSystems/*.py - diracx: name: DiracX Unit Tests runs-on: ubuntu-latest diff --git a/.github/workflows/pilotWrapper.yml b/.github/workflows/pilotWrapper.yml index 305372ee627..451363ec60a 100644 --- a/.github/workflows/pilotWrapper.yml +++ b/.github/workflows/pilotWrapper.yml @@ -13,6 +13,7 @@ jobs: python: - 3.6.8 - 3.11.4 + - 3.14.6 pilot_branch: - master - devel diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst index eacb3cfd164..83422e2602d 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst @@ -40,7 +40,7 @@ where: - configure means adding dirac specific configuration files (which, at a minimum, should include the location of a DIRAC configuration service) -A pilot has to run on each and every computing resource type, provided that Python 2.6+ is on the WN. +A pilot has to run on each and every computing resource type, provided that Python 3.6+ is on the WN. The same pilot script can be used everywhere. .. image:: Pilots2.png diff --git a/setup.cfg b/setup.cfg index 6c9e1c1b2bd..b5eddc216bd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -329,10 +329,9 @@ console_scripts = [pycodestyle] # Pep8 codes: # E203 - Whitespace before ':' (spaces should be present for list slices) -# E211 - Whitespace before '(' (black incorrectly formats exec when in Python 2 mode) # E266 - Too many leading '#' for block comment # E402 - module level import not at top of file (for scripts) # W503 - line break before binary operator (stupid, complains after or before...) -ignore = E203, E211, E266, E402, W503 +ignore = E203, E266, E402, W503 # TODO: This should be reduced back to 120 at some point max_line_length=130 diff --git a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py index 23a28944c59..eef7deb9786 100644 --- a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py +++ b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py @@ -41,8 +41,7 @@ def extractRequestFromFileId(fileId): if compressType == "Z": gLogger.info("Compressed request, uncompressing") try: - # Encoding is only required for Python 2 and can be removed when Python 2 support is no longer needed - stub = base64.urlsafe_b64decode(stub.encode()) + stub = base64.urlsafe_b64decode(stub) except Exception as e: gLogger.error("Oops! Plot request is not properly encoded!", str(e)) return S_ERROR(f"Oops! Plot request is not properly encoded!: {str(e)}") diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py index eec7759fe03..4a47e3f93c0 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py @@ -3,9 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import json import re import tempfile diff --git a/src/DIRAC/Resources/Computing/BatchSystems/GE.py b/src/DIRAC/Resources/Computing/BatchSystems/GE.py index 9f98d6d7789..abaefe95b34 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/GE.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/GE.py @@ -14,9 +14,6 @@ SubmitOption = -l ct=6000 """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import subprocess import os diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Host.py b/src/DIRAC/Resources/Computing/BatchSystems/Host.py index cf3dbe69b30..f50ab67c652 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Host.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Host.py @@ -8,10 +8,6 @@ with LocalComputingElement or SSHComputingElement objects """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import glob import shutil diff --git a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py index 802d93b3625..77cf89753db 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py @@ -9,9 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import subprocess import os diff --git a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py index b8a5a39a783..9765e1d9328 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import os import json diff --git a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py index 623535d126a..82c61a5c227 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py @@ -3,10 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import re import subprocess diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py index 20974d3458e..feeab2aa0c0 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import os diff --git a/src/DIRAC/Resources/Computing/BatchSystems/executeBatch.py b/src/DIRAC/Resources/Computing/BatchSystems/executeBatch.py index 34f30958b10..ff6ebf0f29b 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/executeBatch.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/executeBatch.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - ########################################################### # # Simple executor script for Batch class methods. @@ -10,7 +6,7 @@ # batch system class definition. # # NB: This scipt is executed using the local (to the WN) -# python version, so support for py2 and py3 is necessary. +# python version # # 15.11.2014 # Author: A.T. @@ -23,16 +19,8 @@ import sys import json import traceback - try: - from six.moves.urllib.parse import quote as urlquote - from six.moves.urllib.parse import unquote as urlunquote - except ImportError: - try: - from urllib import unquote as urlunquote - from urllib import quote as urlquote - except ImportError: - from urllib.parse import quote as urlquote - from urllib.parse import unquote as urlunquote + from urllib.parse import quote as urlquote + from urllib.parse import unquote as urlunquote # Read options from JSON file diff --git a/src/DIRAC/Resources/Computing/SSHComputingElement.py b/src/DIRAC/Resources/Computing/SSHComputingElement.py index 30ae5ecbc88..ad2f5f4ab5b 100644 --- a/src/DIRAC/Resources/Computing/SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHComputingElement.py @@ -1,4 +1,4 @@ -""" SSH (Virtual) Computing Element +"""SSH (Virtual) Computing Element For a given IP/host it will send jobs directly through ssh @@ -57,6 +57,7 @@ **Code Documentation** """ + import errno import json import os @@ -396,8 +397,7 @@ def __executeHostCommand(self, connection: Connection, command: str, options: di # Execute the batch command with the options file path cmd = ( f"bash --login -c 'python3 {self.sharedArea}/execute_batch {remoteOptionsFile} || " - f"python {self.sharedArea}/execute_batch {remoteOptionsFile} || " - f"python2 {self.sharedArea}/execute_batch {remoteOptionsFile}'" + f"python {self.sharedArea}/execute_batch {remoteOptionsFile}" ) self.log.verbose(f"CE submission command: {cmd}") diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py index a12d3506358..218db3514ac 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -12,8 +12,6 @@ """ -from __future__ import absolute_import, division, print_function - import base64 import bz2 import os @@ -28,20 +26,14 @@ if [ "${current_limit}" = "unlimited" ] || [ "${current_limit}" -gt "${new_limit}" ]; then ulimit -n "${new_limit}" fi -if command -v python &> /dev/null; then - py='python' -elif command -v python3 &> /dev/null; then +if command -v python3 &> /dev/null; then py='python3' -elif command -v python2 &> /dev/null; then - py='python2' +elif command -v python &> /dev/null; then + py='python' fi /usr/bin/env $py << EOF # imports -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - import os import io import stat @@ -63,19 +55,10 @@ import shlex from uuid import uuid1 -try: - # For Python 3.0 and later - from urllib.request import urlopen, HTTPError, URLError - from urllib.parse import urlencode -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, HTTPError, URLError - from urllib import urlencode +from urllib.request import urlopen +from urllib.error import URLError -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO # formatting with microsecond accuracy, (ISO-8601) @@ -93,10 +76,7 @@ def formatTime(self, record, datefmt=None): # formatter = logging.Formatter(fmt='%%(asctime)s UTC %%(levelname)-8s %%(message)s', datefmt='%%Y-%%m-%%d %%H:%%M:%%S') formatter = MicrosecondFormatter('%%(asctime)s %%(levelname)-8s [%%(name)s] %%(message)s') logging.Formatter.converter = time.gmtime -try: - screen_handler = logging.StreamHandler(stream=sys.stdout) -except TypeError: # python2.6 - screen_handler = logging.StreamHandler(strm=sys.stdout) +screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) # add a string buffer handler @@ -204,10 +184,7 @@ def pilotWrapperScript( try: fd = os.open('%(pfName)s', os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IRUSR | stat.S_IWUSR) with io.open(fd, 'wb') as fd: - if sys.version_info < (3,): - fd.write(bz2.decompress(base64.b64decode(\"\"\"%(encodedPf)s\"\"\"))) - else: - fd.write(bz2.decompress(base64.b64decode(b'%(encodedPf)s'))) + fd.write(bz2.decompress(base64.b64decode(b'%(encodedPf)s'))) os.chmod('%(pfName)s', stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) except Exception as x: print(x, file=sys.stderr) @@ -215,7 +192,7 @@ def pilotWrapperScript( shutil.rmtree(pilotWorkingDirectory) sys.exit(3) """ % { - "encodedPf": encodedPf.decode() if hasattr(encodedPf, "decode") else encodedPf, + "encodedPf": encodedPf.decode() if isinstance(encodedPf, bytes) else encodedPf, "pfName": pfName, } @@ -275,35 +252,17 @@ def pilotWrapperScript( # Getting the json, tar, and checksum file try: - - # urllib is different between python 2 and 3 - if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen - from urllib2 import URLError as url_library_URLError - else: - from urllib.request import urlopen as url_library_urlopen - from urllib.error import URLError as url_library_URLError - for fileName in ['checksums.sha512', 'pilot.json', 'pilot.tar']: - # needs to distinguish whether urlopen method contains the 'context' param - # in theory, it should be available from python 2.7.9 - # in practice, some prior versions may be composed of recent urllib version containing the param - if 'context' in url_library_urlopen.__code__.co_varnames: - import ssl - context = ssl.create_default_context() - check_dirs = [ - os.environ.get('X509_CERT_DIR', '/etc/grid-security/certificates'), - "/cvmfs/grid.cern.ch/etc/grid-security/certificates", - ] - for cert_dir in check_dirs: - if cert_dir and os.path.isdir(cert_dir): - context.load_verify_locations(capath=cert_dir) - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10, - context=context) - else: - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10) + import ssl + context = ssl.create_default_context() + check_dirs = [ + os.environ.get('X509_CERT_DIR', '/etc/grid-security/certificates'), + "/cvmfs/grid.cern.ch/etc/grid-security/certificates", + ] + for cert_dir in check_dirs: + if cert_dir and os.path.isdir(cert_dir): + context.load_verify_locations(capath=cert_dir) + remoteFile = urlopen(os.path.join(loc, fileName), timeout=10, context=context) localFile = open(fileName, 'wb') localFile.write(remoteFile.read()) @@ -326,10 +285,14 @@ def pilotWrapperScript( raise # if we get here we break out of the loop of locations break - except (url_library_URLError, Exception) as e: + except URLError as e: print('%%s unreacheable (this is normal!)' %% loc, file=sys.stderr) logger.error('%%s unreacheable (this is normal!)' %% loc) logger.exception(e) + except Exception as e: + print('Generic error recorded)', file=sys.stderr) + logger.error('Generic error recorded') + logger.exception(e) else: print("None of the locations of the pilot files is reachable", file=sys.stderr) diff --git a/tests/CI/run_pilot.sh b/tests/CI/run_pilot.sh index 31b23e7539d..400a8416c76 100755 --- a/tests/CI/run_pilot.sh +++ b/tests/CI/run_pilot.sh @@ -33,12 +33,10 @@ eval "${PILOT_DOWNLOAD_COMMAND}" echo "${PILOT_JSON}" >pilot.json jq /dev/null; then - py='python' -elif command -v python3 &>/dev/null; then +if command -v python3 &>/dev/null; then py='python3' -elif command -v python2 &>/dev/null; then - py='python2' +elif command -v python &>/dev/null; then + py='python' fi # shellcheck disable=SC2086 diff --git a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py index 4374a26749e..2fb7c5ecece 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py @@ -6,8 +6,6 @@ # - starts it # # It should be executed for different versions of python, e.g.: -# - 2.7.x (x < 9) -# - 2.7.x (x >= 9) # - 3.6.x # - 3.11.x # @@ -17,29 +15,18 @@ # python Test_GenerateAndExecutePilotWrapper.py url://to_PilotWrapper.py -from __future__ import absolute_import, division, print_function - -import os +import ssl # pylint: disable=import-error +import subprocess import sys import time -import subprocess # 1) gets the (DIRAC-free) PilotWrapper.py -# urllib is different between python 2 and 3 -if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen # pylint: disable=import-error -else: - from urllib.request import urlopen as url_library_urlopen # pylint: disable=import-error,no-name-in-module - +from urllib.request import urlopen -if sys.version_info >= (2, 7, 9): - import ssl # pylint: disable=import-error - context = ssl._create_unverified_context() - rf = url_library_urlopen(sys.argv[1], context=context) -else: - rf = url_library_urlopen(sys.argv[1]) +context = ssl._create_unverified_context() +rf = urlopen(sys.argv[1], context=context) locc = sys.argv[2] with open("PilotWrapper.py", "wb") as pj: @@ -52,8 +39,7 @@ from PilotWrapper import pilotWrapperScript # pylint: disable=import-error res = pilotWrapperScript( - pilotOptions="-N ce.dirac.org -Q DIRACQUEUE -n DIRAC.CI.ORG --debug", - location=locc + "/,wrong.cern.ch", + pilotOptions="-N ce.dirac.org -Q DIRACQUEUE -n DIRAC.CI.ORG --debug", location="wrong.cern.ch, " + locc ) with open("pilot-wrapper.sh", "wb") as pj: