From 81d5133cba06e0e0ed1e49174ce361322e2a5aec Mon Sep 17 00:00:00 2001 From: Alejandro R Mosteo Date: Tue, 23 Jan 2024 13:50:20 +0100 Subject: [PATCH] Use e3 facilities for test skipping (#1537) --- testsuite/README.md | 2 +- testsuite/drivers/driver/base_driver.py | 54 +++++++++++++++++++ testsuite/drivers/driver/docker_nested.py | 4 +- testsuite/drivers/driver/docker_wrapper.py | 11 ++-- testsuite/drivers/driver/python_script.py | 10 ++-- testsuite/drivers/helpers.py | 6 --- testsuite/run.py | 6 ++- testsuite/skels/no-index/test.yaml | 9 +++- .../dockerized/misc/default-cache/test.yaml | 2 + .../misc/failed-auto-update/test.yaml | 3 ++ .../tests/index/external-msys2/test.yaml | 2 + 11 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 testsuite/drivers/driver/base_driver.py diff --git a/testsuite/README.md b/testsuite/README.md index 28b3a50f..484ce4c5 100644 --- a/testsuite/README.md +++ b/testsuite/README.md @@ -43,7 +43,7 @@ All tests are based on running a Python script. There are these test drivers: - `docker-wrapper`: run in a pristine docker Ubuntu image in shared build mode. # Environment variables -The following variables can be used to modify test/testsuite behavior. +The following variables can be used to modify testsuite behavior. For `ALIRE_DISABLE_*` variables, their mere existence activates their function, no matter their value, or lack of one. diff --git a/testsuite/drivers/driver/base_driver.py b/testsuite/drivers/driver/base_driver.py new file mode 100644 index 00000000..522c3aea --- /dev/null +++ b/testsuite/drivers/driver/base_driver.py @@ -0,0 +1,54 @@ +import os +from typing import Any, Dict +import e3 + +import drivers.helpers + +from e3.testsuite.control import YAMLTestControlCreator +from e3.testsuite.driver.classic import ClassicTestDriver + + +class BaseDriver(ClassicTestDriver): + """ + A common base driver from which the rest inherit, so common functionality + can be shared by all drivers. In particular, enabling/disabling via the e3 + control feature. + """ + + # Environment variables that can be used to modify the testsuite behavior + # in the control field of the test YAML file. + # E.g.: + # control: + # - [SKIP, "skip_docker", "Docker is disabled"] + MODIFIERS = { + "distro" : 'ALIRE_DISABLE_DISTRO' + , "docker" : 'ALIRE_DISABLE_DOCKER' + , "local" : 'ALIRE_ENABLE_LOCAL_TESTS' + , "network" : 'ALIRE_DISABLE_NETWORK_TESTS' + } + + # In the constructor, prepare the features map based on the environment + # variables. + def __init__(self, env: e3.env.Env, test_env: Dict[str, Any]): + super().__init__(env, test_env) + + # Prepare the features map based on the environment variables + self.skip = {} + + for modifier, env_var in BaseDriver.MODIFIERS.items(): + affirming = "ENABLE" in env_var + key = f"skip_{modifier}" + + if env_var in os.environ: + self.skip[key] = not affirming + else: + self.skip[key] = affirming + + # Hardcode OSes + for osname in ["windows", "linux", "macos"]: + self.skip[f"skip_{osname}"] = not getattr(drivers.helpers, f"on_{osname}")() + + @property + def test_control_creator(self): + # Give our custom dictionary to the control filter + return YAMLTestControlCreator(self.skip) diff --git a/testsuite/drivers/driver/docker_nested.py b/testsuite/drivers/driver/docker_nested.py index 647ab650..cce50544 100644 --- a/testsuite/drivers/driver/docker_nested.py +++ b/testsuite/drivers/driver/docker_nested.py @@ -10,7 +10,7 @@ import sys from drivers import alr from drivers.alr import run_alr -from drivers.helpers import MODIFIERS +from drivers.driver.base_driver import BaseDriver def main(): @@ -36,7 +36,7 @@ def main(): # Set up the environment # Copy any received modifiers - for modifier in [m for m in MODIFIERS if m in test_env]: + for modifier in [m for m in BaseDriver.MODIFIERS.items() if m in test_env]: os.environ[modifier] = test_env[modifier] # alr path diff --git a/testsuite/drivers/driver/docker_wrapper.py b/testsuite/drivers/driver/docker_wrapper.py index 5d974a72..3a6c2055 100644 --- a/testsuite/drivers/driver/docker_wrapper.py +++ b/testsuite/drivers/driver/docker_wrapper.py @@ -5,9 +5,10 @@ import subprocess from importlib import import_module from typing import Tuple -from drivers.helpers import FileLock, on_linux, MODIFIERS -from e3.testsuite.driver.classic import (ClassicTestDriver, - TestAbortWithFailure, +from drivers.driver.base_driver import BaseDriver +from drivers.helpers import FileLock, on_linux + +from e3.testsuite.driver.classic import (TestAbortWithFailure, TestSkip) DOCKERFILE = "Dockerfile" @@ -88,7 +89,7 @@ def build_image() -> None: labels={LABEL_HASH : compute_dockerfile_hash()}) -class DockerWrapperDriver(ClassicTestDriver): +class DockerWrapperDriver(BaseDriver): # This is a workaround for Windows, where attempting to use rlimit by e3-core # causes permission errors. TODO: remove once e3-core has a proper solution. @@ -104,7 +105,7 @@ class DockerWrapperDriver(ClassicTestDriver): # Augment the test environment with local modifiers that could # impact the wrapped test, if defined - for modifier in [m for m in MODIFIERS if m in os.environ]: + for modifier in [val for _, val in self.MODIFIERS.items() if val in os.environ]: self.test_env[modifier] = os.environ[modifier] # Run our things diff --git a/testsuite/drivers/driver/python_script.py b/testsuite/drivers/driver/python_script.py index b646b301..8e4e25a6 100644 --- a/testsuite/drivers/driver/python_script.py +++ b/testsuite/drivers/driver/python_script.py @@ -4,12 +4,13 @@ import shutil import sys from drivers.alr import prepare_env, prepare_indexes, run_alr -from e3.testsuite.driver.classic import (ClassicTestDriver, - TestAbortWithFailure, +from drivers.driver.base_driver import BaseDriver + +from e3.testsuite.driver.classic import (TestAbortWithFailure, TestSkip) -class PythonScriptDriver(ClassicTestDriver): +class PythonScriptDriver(BaseDriver): """ Test driver to run a "test.py" Python script. @@ -142,6 +143,9 @@ class PythonScriptDriver(ClassicTestDriver): self.test_env.get('build-mode', DEFAULT_MODE)) # One of 'shared', 'sandboxed', or 'both' + if mode not in ["shared", "sandboxed", "both"]: + raise ValueError(f"Invalid build mode: {mode}, must be one of " + "'shared', 'sandboxed', or 'both'") # If mode is "both", track original files for later if mode == "both": diff --git a/testsuite/drivers/helpers.py b/testsuite/drivers/helpers.py index 9d0b2e64..b89ff9c1 100644 --- a/testsuite/drivers/helpers.py +++ b/testsuite/drivers/helpers.py @@ -11,12 +11,6 @@ import stat from subprocess import run from zipfile import ZipFile -# Environment variables that can be used to modify the testsuite behavior -MODIFIERS : list = [ - 'ALIRE_DISABLE_DISTRO' - , 'ALIRE_DISABLE_DOCKER' - , 'ALIRE_DISABLE_NETWORK_TESTS' -] # Return the entries (sorted) under a given folder, both folders and files # Optionally, return only those matching regex. Uses '/' always as separator. diff --git a/testsuite/run.py b/testsuite/run.py index 8254dbba..15f9370c 100755 --- a/testsuite/run.py +++ b/testsuite/run.py @@ -22,9 +22,11 @@ from drivers.helpers import on_windows class Testsuite(e3.testsuite.Testsuite): tests_subdir = 'tests' + + # Available drivers test_driver_map = { - 'python-script': PythonScriptDriver, - 'docker-wrapper': DockerWrapperDriver + 'python-script' : PythonScriptDriver, + 'docker-wrapper' : DockerWrapperDriver } def add_options(self, parser): diff --git a/testsuite/skels/no-index/test.yaml b/testsuite/skels/no-index/test.yaml index 45e528c2..a4f7b6b2 100644 --- a/testsuite/skels/no-index/test.yaml +++ b/testsuite/skels/no-index/test.yaml @@ -1,7 +1,14 @@ driver: python-script build_mode: both # one of shared, sandboxed, both (default) +control: # Used to disable test depending on one of: + - [SKIP, "skip_distro", "Unknown distro testing disabled"] + - [SKIP, "skip_docker", "Docker-hosted tests disabled"] + - [SKIP, "skip_network", "Network-requiring tests disabled"] + - [SKIP, "skip_linux", "Test is Linux-only"] + - [SKIP, "skip_macos", "Test is macOS-only"] + - [SKIP, "skip_windows", "Test is Windows-only"] indexes: compiler_only_index: {} # Note that shared builds require a detected compiler to be able to compute # build hashes, which is needed for many subcommands: build, get, printenv, - # update... \ No newline at end of file + # update... diff --git a/testsuite/tests/dockerized/misc/default-cache/test.yaml b/testsuite/tests/dockerized/misc/default-cache/test.yaml index 38ab62d3..c93da6ef 100644 --- a/testsuite/tests/dockerized/misc/default-cache/test.yaml +++ b/testsuite/tests/dockerized/misc/default-cache/test.yaml @@ -1,4 +1,6 @@ driver: docker-wrapper +control: + - [SKIP, "skip_docker", "Docker disabled"] indexes: toolchain_index: copy_crates_src: True diff --git a/testsuite/tests/dockerized/misc/failed-auto-update/test.yaml b/testsuite/tests/dockerized/misc/failed-auto-update/test.yaml index d0556473..3ddc0289 100644 --- a/testsuite/tests/dockerized/misc/failed-auto-update/test.yaml +++ b/testsuite/tests/dockerized/misc/failed-auto-update/test.yaml @@ -1,2 +1,5 @@ driver: docker-wrapper +control: + - [SKIP, "skip_docker", "Docker disabled"] + - [SKIP, "skip_network", "Network disabled"] indexes: {} diff --git a/testsuite/tests/index/external-msys2/test.yaml b/testsuite/tests/index/external-msys2/test.yaml index b7da6fe7..d38932f6 100644 --- a/testsuite/tests/index/external-msys2/test.yaml +++ b/testsuite/tests/index/external-msys2/test.yaml @@ -1,4 +1,6 @@ driver: python-script +control: + - [SKIP, "skip_windows", "Windows-only"] indexes: my_index: in_fixtures: false -- 2.39.5