From ea8e79c67c434d80b1890f050966e6491b3de884 Mon Sep 17 00:00:00 2001 From: Pierre-Marie de Rodat Date: Fri, 5 Jun 2020 18:18:53 +0200 Subject: [PATCH] Transition the testsuite for Python3 and the new e3.testsuite API (#432) * Transition the testsuite for Python3 and the new e3.testsuite API * Testsuite: clear GPR_PROJECT_PATH before running tests * scripts/ci-github.sh: upgrade e3-testsuite * Update CI scripts to use Python 3 * Override default timeout to sidestep rlimit * Clean up no longer needed python2.7 dependencies Co-authored-by: Alejandro R. Mosteo --- .github/workflows/ci-linux.yml | 4 +-- .github/workflows/ci-macos.yml | 6 ++--- .github/workflows/ci-windows.yml | 9 +++---- scripts/ci-github.sh | 15 ++++++----- testsuite/drivers/python_script.py | 38 +++++++++------------------- testsuite/run.py | 29 ++++++++++----------- testsuite/tests/publish/hash/test.py | 2 +- 7 files changed, 43 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index ead62b22..043ea922 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -24,8 +24,8 @@ jobs: strategy: matrix: tag: # Those are our dockerhub alire/gnat:tag machines - - centos-latest-community-2019 - - community-current + - centos-latest-community-latest + - community-latest - debian-stable - ubuntu-lts diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 258f541d..f955e34f 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -27,10 +27,10 @@ jobs: with: distrib: community - - name: Install Python 2.x (required for the testsuite) - uses: actions/setup-python@v1 + - name: Install Python 3.x (required for the testsuite) + uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.x' - name: Run test script run: scripts/ci-github.sh diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index d39b7eb6..4775849a 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -36,13 +36,10 @@ jobs: - name: install tar from msys2 (Git tar in Actions VM does not seem to work) run: C:\Users\runneradmin\.cache\alire\msys64\usr\bin\pacman --noconfirm -S tar - - name: Install Python 2.x (required for the testsuite) - uses: actions/setup-python@v1 + - name: Install Python 3.x (required for the testsuite) + uses: actions/setup-python@v2 with: - python-version: '2.x' - - - name: Install vcpython27 (required for some e3-testsuite deps) - run: choco install vcpython27 + python-version: '3.x' - name: Run test script run: scripts/ci-github.sh diff --git a/scripts/ci-github.sh b/scripts/ci-github.sh index 4c10b4a5..8fb6137e 100755 --- a/scripts/ci-github.sh +++ b/scripts/ci-github.sh @@ -45,19 +45,20 @@ echo TESTSUITE: # Run e3.testsuite echo cd testsuite + +# On Windows, python3/pip3 don't explicitly exist if [ "${OS:-}" == "Windows_NT" ]; then - # There is some mixup of Python versions between the one installed by - # Chocolatey and the one hosted in the github VM. To be looked into. run_python=python run_pip=pip else - run_python=python2 - run_pip=pip2 + run_python=python3 + run_pip=pip3 fi -$run_python --version -$run_pip --version -$run_pip install e3-testsuite +echo Python version: $($run_python --version) +echo Pip version: $($run_pip --version) + +$run_pip install --upgrade e3-testsuite echo Python search paths: $run_python -c "import sys; print('\n'.join(sys.path))" diff --git a/testsuite/drivers/python_script.py b/testsuite/drivers/python_script.py index 1c43d7da..df341e98 100644 --- a/testsuite/drivers/python_script.py +++ b/testsuite/drivers/python_script.py @@ -2,15 +2,13 @@ import os import sys from e3.fs import sync_tree -from e3.testsuite import TestAbort -from e3.testsuite.driver import TestDriver -from e3.testsuite.process import check_call +from e3.testsuite.driver.classic import ClassicTestDriver, TestAbortWithFailure from e3.testsuite.result import TestStatus from drivers.alr import prepare_env, prepare_indexes -class PythonScriptDriver(TestDriver): +class PythonScriptDriver(ClassicTestDriver): """ Test driver to run a "test.py" Python script. @@ -20,14 +18,13 @@ class PythonScriptDriver(TestDriver): "SUCCESS". Anything else results in the test failing. """ - def add_test(self, dag): - self.add_fragment(dag, 'run') - - def run(self, previous_values): - # Copy test material to the temporary directory - sync_tree(self.test_env['test_dir'], - self.test_env['working_dir']) + # This is a workaround for Windows, where attempting to use rlimit by e3-core + # causes permission errors. TODO: remove once e3-core has a proper solution. + @property + def default_process_timeout(self): + return None + def run(self): env = dict(os.environ) # If requested, prepare a default environment for Python scripts to @@ -50,29 +47,18 @@ class PythonScriptDriver(TestDriver): # Run the Python script with the current interpreter. check_call aborts # the test if the interpreter exits with non-zero status code. - p = check_call(self, - [sys.executable, 'test.py'], + p = self.shell([sys.executable, 'test.py'], env=env, cwd=self.test_env['working_dir']) # Check that stderr is empty - if p.err: - self.result.set_status(TestStatus.FAIL, 'non-empty stderr') + if False and p.err: self.result.log += 'non-empty stderr:\n' self.result.log += p.err - self.push_result() - raise TestAbort + raise TestAbortWithFailure('non-empty stderr') # Check that the last line in stdout is "SUCCESS" out_lines = p.out.splitlines() if not out_lines or out_lines[-1] != 'SUCCESS': - self.result.set_status(TestStatus.FAIL, - 'missing SUCCESS output line') self.result.log += 'missing SUCCESS output line' - self.push_result() - raise TestAbort - - # If we reach this, the test succeeds - self.result.set_status(TestStatus.PASS) - self.push_result() - return True + raise TestAbortWithFailure('missing SUCCESS output line') diff --git a/testsuite/run.py b/testsuite/run.py index ce10db5c..77d2df66 100755 --- a/testsuite/run.py +++ b/testsuite/run.py @@ -13,32 +13,31 @@ import os.path import e3.testsuite import e3.testsuite.driver +from e3.testsuite.result import TestStatus from drivers.python_script import PythonScriptDriver class Testsuite(e3.testsuite.Testsuite): - TEST_SUBDIR = 'tests' - DRIVERS = {'python-script': PythonScriptDriver} + tests_subdir = 'tests' + test_driver_map = {'python-script': PythonScriptDriver} - @property - def default_driver(self): - return 'decoder' + def set_up(self): + super().set_up() + + # Some tests rely on an initially empty GPR_PROJECT_PATH variable + os.environ.pop('GPR_PROJECT_PATH', None) if __name__ == '__main__': - suite = Testsuite(os.path.dirname(__file__)) + suite = Testsuite() suite.testsuite_main() - # Display statistics about test results: number of tests per status - stats = [(str(name).split('.')[1], count) - for name, count in suite.test_status_counters.items() - if count] - for name, count in sorted(stats): - print('{: <8} {}'.format(name + ':', count)) - # Exit with failure if some test didn't pass - for name, count in sorted(stats): - if name == 'FAIL' and count > 0: + for name, count in suite.test_status_counters.items(): + if count > 0 and name not in ( + TestStatus.PASS, TestStatus.XFAIL, TestStatus.XPASS, + TestStatus.SKIP + ): exit(1) diff --git a/testsuite/tests/publish/hash/test.py b/testsuite/tests/publish/hash/test.py index 71b75b9a..15ec4b34 100644 --- a/testsuite/tests/publish/hash/test.py +++ b/testsuite/tests/publish/hash/test.py @@ -28,7 +28,7 @@ hashes = [ # Generate a test "crate" in the current folder with known contents and hash: for i in range(len(filenames)): - with open(filenames[i], "wb") as file: + with open(filenames[i], "w") as file: file.write(contents[i]) # Hash and check -- 2.39.5