From ff7048f0b12551b2e9f133b0a8fde0393dc0f4fa Mon Sep 17 00:00:00 2001 From: Alejandro R Mosteo Date: Sun, 23 Jun 2024 15:22:35 +0200 Subject: [PATCH] fix: test runs with recent Docker images (#1704) * fix: git warnings during test runs * Fix semver parsing in APT and From_Output * Fix ownership of local git repo used during tests Using the --shared feature of init * Fix toolchain workflow to ensure use of intended toolchain --- .github/workflows/ci-docker.yml | 2 +- .github/workflows/ci-toolchain.yml | 27 ++++++- alire.toml | 10 +-- deps/semantic_versioning | 2 +- dev/build.sh | 4 +- scripts/ci-github.sh | 23 ++++-- scripts/version-patcher.sh | 2 +- src/alire/alire-externals-from_output.adb | 29 ++++--- src/alire/alire-externals-from_system.adb | 71 +++++++++++------- .../alire-origins-deployers-system-apt.adb | 13 ++++ testsuite/fixtures/crates/libfoo_git/HEAD | 2 +- testsuite/fixtures/crates/libfoo_git/config | 3 + .../07/2d593793eda4efbb0af13f3fa422d72f5492d3 | Bin 18 -> 0 bytes .../9a/6aa1471db1a0c821570db875b12f08653f623c | 1 - .../f4/fdb928df3b0be3f896b18be4987ce7338f0a54 | Bin 127 -> 0 bytes .../fixtures/crates/libfoo_git/packed-refs | 2 - testsuite/fixtures/crates/libhello_git | 1 - .../my_index/crates/crate}/.emptydir | 0 .../index/cr/crate1/crate1-external.toml | 10 +++ .../index/cr/crate2/crate2-external.toml | 10 +++ .../bad-versions/my_index/index/index.toml | 1 + .../tests/externals/bad-versions/test.py | 18 +++++ .../tests/externals/bad-versions/test.yaml | 5 ++ 23 files changed, 167 insertions(+), 69 deletions(-) delete mode 100644 testsuite/fixtures/crates/libfoo_git/objects/07/2d593793eda4efbb0af13f3fa422d72f5492d3 delete mode 100644 testsuite/fixtures/crates/libfoo_git/objects/9a/6aa1471db1a0c821570db875b12f08653f623c delete mode 100644 testsuite/fixtures/crates/libfoo_git/objects/f4/fdb928df3b0be3f896b18be4987ce7338f0a54 delete mode 100644 testsuite/fixtures/crates/libfoo_git/packed-refs delete mode 160000 testsuite/fixtures/crates/libhello_git rename testsuite/{fixtures/crates/libfoo_git/refs/heads => tests/externals/bad-versions/my_index/crates/crate}/.emptydir (100%) create mode 100644 testsuite/tests/externals/bad-versions/my_index/index/cr/crate1/crate1-external.toml create mode 100644 testsuite/tests/externals/bad-versions/my_index/index/cr/crate2/crate2-external.toml create mode 100644 testsuite/tests/externals/bad-versions/my_index/index/index.toml create mode 100644 testsuite/tests/externals/bad-versions/test.py create mode 100644 testsuite/tests/externals/bad-versions/test.yaml diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 6808694c..a01ad2d0 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/ci-toolchain.yml b/.github/workflows/ci-toolchain.yml index 57bfb76a..e68bac79 100644 --- a/.github/workflows/ci-toolchain.yml +++ b/.github/workflows/ci-toolchain.yml @@ -1,6 +1,10 @@ name: CI self+toolchain # Build Alire with `alr build` and using a toolchain installed from Alire # The `alr` being tested is the one which is being submitted in the PR +# Toolchain is tested from `alr install` (1st build) and `alr build` (2nd +# build), so those are two different toolchain installations of the same +# toolchain. This way we not only test that alr builds itself, but that +# toolchain installations via `alr install` work as intented. on: pull_request: @@ -40,12 +44,22 @@ jobs: with: crates: gnat_native^${{matrix.gcc_version}} gprbuild - - name: Build alr with default toolchain + - name: Build alr with toolchain from `alr install` shell: bash run: dev/build.sh # We can start using the alr we just built + - name: Select toolchain GNAT^${{matrix.gcc_version}} + run: ./bin/alr -d -n toolchain --select gnat_native^${{matrix.gcc_version}} gprbuild + + - name: Show builder alr configuration + run: ./bin/alr -d -n version + + - name: Verify proper toolchain used for 1st build + shell: bash + run: ./bin/alr -d -n version | grep 'compiled with version' | grep -q '${{ matrix.gcc_version }}' + - name: Update dependencies run: ./bin/alr -d -n update @@ -55,7 +69,8 @@ jobs: - name: Show build environment, with debug fallback run: ./bin/alr -d -n printenv || ./bin/alr -n -v -d printenv - - shell: bash + - name: Move ./bin to ./bin-old to allow for self-build + shell: bash run: mv ./bin ./bin-old || { sleep 5s && mv ./bin ./bin-old; } # Windows doesn't allow to replace a running exe so the next command # fails otherwise. Also, this mv fails sometimes so we try twice JIC. @@ -67,6 +82,10 @@ jobs: - name: Show built version run: ./bin/alr -d -n version + - name: Verify proper toolchain used for 2nd build + shell: bash + run: ./bin/alr -d -n version | grep 'compiled with version' | grep -q '${{ matrix.gcc_version }}' + # Run the testsuite with the just build alr. The testsuite picks the proper # alr in the ./bin/alr location. @@ -78,6 +97,6 @@ jobs: - name: Install e3 run: pip install --upgrade -r testsuite/requirements.txt - - name: Run testsuite - run: cd testsuite; ./run.py -E + - name: Run testsuite # But ensure a new alr is not build + run: scripts/ci-github.sh build=false shell: bash diff --git a/alire.toml b/alire.toml index 0165eeb7..a928140f 100644 --- a/alire.toml +++ b/alire.toml @@ -61,11 +61,11 @@ commit = "56bbdc008e16996b6f76e443fd0165a240de1b13" [pins.dirty_booleans] url = "https://github.com/mosteo/dirty_booleans" -branch = "alire" +commit = "05c40d88ecfe109e575ec8b21dd6ffa2e61df1dc" [pins.diskflags] url = "https://github.com/mosteo/diskflags" -branch = "alire" +commit = "60729edf31816aca0036b13b2794c39a9bd0172e" [pins.gnatcoll] url = "https://github.com/alire-project/gnatcoll-core.git" @@ -77,7 +77,7 @@ commit = "9a9c660f9c6f27f5ef75417e7fac7061dff14d78" [pins.semantic_versioning] url = "https://github.com/alire-project/semantic_versioning" -commit = "cc2148cf9c8934fb557b5ae49a3f7947194fa7ee" +commit = "4861e32bd8a2f0df038d3ecc9a72b6381e7a34cc" [pins.simple_logging] url = "https://github.com/alire-project/simple_logging" @@ -85,7 +85,7 @@ commit = "3505dc645f3eef6799a486aae223d37e88cfc4d5" [pins.si_units] url = "https://github.com/mosteo/si_units" -branch = "alire" +commit = "9329d2591b82440ccc859a53f1380ac07ea4194d" [pins.stopwatch] url = "https://github.com/mosteo/stopwatch" @@ -93,7 +93,7 @@ commit = "f607a63b714f09bbf6126de9851cbc21cf8666c9" [pins.toml_slicer] url = "https://github.com/mosteo/toml_slicer" -branch = "alire" +commit = "3e5cbdb5673b85a1da6344a41764ef1cbafe3289" # To disable version updating, export ALR_VERSION_DONT_PATCH with any value diff --git a/deps/semantic_versioning b/deps/semantic_versioning index cc2148cf..4861e32b 160000 --- a/deps/semantic_versioning +++ b/deps/semantic_versioning @@ -1 +1 @@ -Subproject commit cc2148cf9c8934fb557b5ae49a3f7947194fa7ee +Subproject commit 4861e32bd8a2f0df038d3ecc9a72b6381e7a34cc diff --git a/dev/build.sh b/dev/build.sh index 93498640..336f043c 100755 --- a/dev/build.sh +++ b/dev/build.sh @@ -10,7 +10,7 @@ ALIRE_OS=$(get_OS); export ALIRE_OS scripts/version-patcher.sh -echo "Building with ALIRE_OS=$ALIRE_OS..." +echo "Building with ALIRE_OS=$ALIRE_OS and $(gnat --version | head -1)" gprbuild "-j$ALIRE_BUILD_JOBS" -r -p -P "$(dirname $0)/../alr_env.gpr" "$@" -scripts/version-patcher.sh _or_later \ No newline at end of file +scripts/version-patcher.sh _or_later diff --git a/scripts/ci-github.sh b/scripts/ci-github.sh index 845cd96d..06800270 100755 --- a/scripts/ci-github.sh +++ b/scripts/ci-github.sh @@ -13,18 +13,29 @@ pushd "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" . ../dev/functions.sh popd -# Mark location safe to assuage git if necessary (happens in some distros) +# Mark location safe to assuage git if necessary (happens under docker as we +# run with a different user). if git status 2>&1 | grep -q "dubious ownership"; then - echo "Marking $PWD as safe for git" - git config --global --add safe.directory "$PWD" + echo "Marking $PWD as safe for git" + git config --global --add safe.directory "$PWD" + + # Change ownership and group to current user of everything in the testsuite, + # as we have there some pre-created git repositories that would fail too. + # These are copied to temporary locations by the test runner, so we cannot + # simply use the `git config` trick. + sudo chown -R $(id -u):$(id -g) testsuite fi # Patch version scripts/version-patcher.sh -# Build alr -export ALIRE_OS=$(get_OS) -gprbuild -j0 -p -P alr_env +# Build alr if no argument is "build=false" +if [[ " $* " == *" build=false "* ]]; then + echo "Skipping alr build, explicitly disabled via arguments" +else + export ALIRE_OS=$(get_OS) + gprbuild -j0 -p -P alr_env +fi # Disable distro detection if supported if [ "${ALIRE_DISABLE_DISTRO:-}" == "true" ]; then diff --git a/scripts/version-patcher.sh b/scripts/version-patcher.sh index 1b0c0789..9e666d56 100755 --- a/scripts/version-patcher.sh +++ b/scripts/version-patcher.sh @@ -28,4 +28,4 @@ fi $bin "$@" echo "Resulting version file:" -cat src/alire/alire-version.ads | grep Current_Str \ No newline at end of file +cat src/alire/alire-version.ads | grep "Current_Str : constant String" \ No newline at end of file diff --git a/src/alire/alire-externals-from_output.adb b/src/alire/alire-externals-from_output.adb index 97c8b7df..965b0757 100644 --- a/src/alire/alire-externals-from_output.adb +++ b/src/alire/alire-externals-from_output.adb @@ -7,6 +7,7 @@ with Alire.OS_Lib.Subprocess; with Alire.Paths; with Alire.Releases; with Alire.TOML_Keys; +with Alire.Utils.Regex; with Alire.Utils.TTY; with Semantic_Versioning; @@ -48,20 +49,13 @@ package body Alire.Externals.From_Output is end if; declare - use GNAT.Regpat; - Matches : Match_Array (1 .. Match_Count'Last); - -- Although we should have at most one match, it turns out that the - -- match won't necessarily be on the first position in the array, as - -- it depends on the number of () expressions. - Lines : AAA.Strings.Vector; Status : constant Integer := OS_Lib.Subprocess.Unchecked_Spawn_And_Capture (This.Command.First_Element, This.Command.Tail, Lines); - Output : constant String := - Lines.Flatten ("" & ASCII.LF); + Output : constant String := Lines.Flatten ("" & ASCII.LF); -- ASCII.LF is used by Regpat for new lines begin if Status /= 0 then @@ -73,13 +67,13 @@ package body Alire.Externals.From_Output is Trace.Debug ("Looking for external in version string '" & Output & "'"); - Match (This.Regexp, Output, Matches); - for I in Matches'Range loop - if Matches (I) /= No_Match then + declare + Version : constant String := + Utils.Regex.First_Match (+This.Regstr, Output); + begin + if Version /= "" then declare - Version : constant String := - Output (Matches (I).First .. Matches (I).Last); Path : constant Any_Path := OS_Lib.Subprocess.Locate_In_Path (This.Command.First_Element); @@ -89,7 +83,8 @@ package body Alire.Externals.From_Output is Result.Insert (Index.Crate (Name, Index.Query_Mem_Only).Base - .Retagging (Semantic_Versioning.Parse (Version)) + .Retagging (Semantic_Versioning.Parse + (Version, Relaxed => True)) .Providing (This.Provides) .Replacing (Origins.New_External ("path " & Path)) .Replacing (Notes => "Detected at " -- length is 12 @@ -97,8 +92,12 @@ package body Alire.Externals.From_Output is (String (Path), Max_Description_Length - 12))); end; + else + Trace.Debug + ("Failed to match a version using regexp '" + & (+This.Regstr) & "' on output: " & Version); end if; - end loop; + end; end; return Result; diff --git a/src/alire/alire-externals-from_system.adb b/src/alire/alire-externals-from_system.adb index b0edceed..88bdfabc 100644 --- a/src/alire/alire-externals-from_system.adb +++ b/src/alire/alire-externals-from_system.adb @@ -39,36 +39,49 @@ package body Alire.Externals.From_System is else for Candidate of Origin.Value.Packages loop Trace.Detail ("Looking for system package: " & Candidate); - declare - Detector : constant System.Deployer'Class := - System.Platform_Deployer (Candidate); - Result : constant System.Version_Outcomes.Outcome := - Detector.Detect; begin - if Result.Success then - Trace.Detail ("Success with system package: " - & Candidate); - - -- A system package may be found more than once if - -- transitional and proper package names are given - -- for detection of the same system package. - declare - Release : constant Alire.Releases.Release - := Index.Crate (Name, Index.Query_Mem_Only).Base - .Retagging (Result.Value) - .Providing (This.Provides) - .Replacing (Origins.New_System (Candidate)) - .Replacing (Notes => "Provided by system package:" - & " " & Candidate); - begin - if Releases.Contains (Release) then - Trace.Debug ("Not readding same system package: " - & Release.Milestone.Image); - else - Releases.Insert (Release); - end if; - end; - end if; + declare + Detector : constant System.Deployer'Class := + System.Platform_Deployer (Candidate); + Result : constant System.Version_Outcomes.Outcome := + Detector.Detect; + begin + if Result.Success then + Trace.Detail ("Success with system package: " + & Candidate); + + -- A system package may be found more than once if + -- transitional and proper package names are given + -- for detection of the same system package. + declare + Release : constant Alire.Releases.Release + := Index.Crate (Name, + Index.Query_Mem_Only) + .Base + .Retagging (Result.Value) + .Providing (This.Provides) + .Replacing (Origins.New_System (Candidate)) + .Replacing + (Notes => "Provided by system package:" + & " " & Candidate); + begin + if Releases.Contains (Release) then + Trace.Debug + ("Not readding same system package: " + & Release.Milestone.Image); + else + Releases.Insert (Release); + end if; + end; + end if; + end; + exception + when E : others => + Trace.Debug + ("Error attempting version detection of system " + & "package [" & Candidate & "] for crate: " + & (+Name)); + Log_Exception (E); end; end loop; end if; diff --git a/src/alire/alire-origins-deployers-system-apt.adb b/src/alire/alire-origins-deployers-system-apt.adb index 4f4db6ff..2207482f 100644 --- a/src/alire/alire-origins-deployers-system-apt.adb +++ b/src/alire/alire-origins-deployers-system-apt.adb @@ -77,6 +77,19 @@ package body Alire.Origins.Deployers.System.Apt is Trace.Debug ("Unexpected version format, could not identify version"); end if; + exception + -- We do not really want to disturb users for a problem + -- introduced externally by some new package version in the + -- underlying distro. This will make the problem harder to + -- detect, but eventually someone should notice that a package + -- is not being detected as intended. + when Constraint_Error | Semantic_Versioning.Malformed_Input => + Trace.Debug + ("Unexpected error while parsing version from: " + & Match & " in line " & Line & " in pkg " + & This.Base.Package_Name); + return Version_Outcomes.Outcome_Failure + ("could not be detected", Report => False); end; end if; end loop; diff --git a/testsuite/fixtures/crates/libfoo_git/HEAD b/testsuite/fixtures/crates/libfoo_git/HEAD index cb089cd8..b870d826 100644 --- a/testsuite/fixtures/crates/libfoo_git/HEAD +++ b/testsuite/fixtures/crates/libfoo_git/HEAD @@ -1 +1 @@ -ref: refs/heads/master +ref: refs/heads/main diff --git a/testsuite/fixtures/crates/libfoo_git/config b/testsuite/fixtures/crates/libfoo_git/config index 07d359d0..164a3ce2 100644 --- a/testsuite/fixtures/crates/libfoo_git/config +++ b/testsuite/fixtures/crates/libfoo_git/config @@ -2,3 +2,6 @@ repositoryformatversion = 0 filemode = true bare = true + sharedrepository = 2 +[receive] + denyNonFastforwards = true diff --git a/testsuite/fixtures/crates/libfoo_git/objects/07/2d593793eda4efbb0af13f3fa422d72f5492d3 b/testsuite/fixtures/crates/libfoo_git/objects/07/2d593793eda4efbb0af13f3fa422d72f5492d3 deleted file mode 100644 index eae9323f7044afd2a8f7476e1c645694655608bf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18 ZcmbV$TF_24NNEL;yc01-SqK diff --git a/testsuite/fixtures/crates/libfoo_git/objects/9a/6aa1471db1a0c821570db875b12f08653f623c b/testsuite/fixtures/crates/libfoo_git/objects/9a/6aa1471db1a0c821570db875b12f08653f623c deleted file mode 100644 index 30748e72..00000000 --- a/testsuite/fixtures/crates/libfoo_git/objects/9a/6aa1471db1a0c821570db875b12f08653f623c +++ /dev/null @@ -1 +0,0 @@ -x¥Í½ @agžâî& Z.Icttpñ ø¹DL„Òŧ7©àv¦ïøšsê Œ<õFQÅà̈!JÇɈfv)ƒÚ“–#·“bvï¯Úà¶ÒۖÐ*<xÔ­S…Åæ#®{IÛÚ. &Í*=#œùÈ9óǸÓ»—Ô“]ág±/žÔ?µ \ No newline at end of file diff --git a/testsuite/fixtures/crates/libfoo_git/objects/f4/fdb928df3b0be3f896b18be4987ce7338f0a54 b/testsuite/fixtures/crates/libfoo_git/objects/f4/fdb928df3b0be3f896b18be4987ce7338f0a54 deleted file mode 100644 index 64745dc27b2f42bfd85ff3765f1d55806a6cf37f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 127 zcmV-_0D%8^0V^p=O;s>7G-EI{FfcPQQAlK{m?5c@Xsz`-L6t@S+uruBgEReFO$>lQ zA&KF1px^54uIoSTTq=6z$&p*D)`Y!X57n5=@NDj**_$q2bN24mTvr}-