From 9b476d0ea62e8ce72a97c602ecd5f5e650724fc7 Mon Sep 17 00:00:00 2001 From: Alejandro R Mosteo Date: Sat, 5 Aug 2023 21:10:44 +0200 Subject: [PATCH] New shared builds mechanism (mockup) (#1419) * New shared builds mechanism (mockup) * Self-review * Fixes for Windows --- doc/user-changes.md | 29 ++++ src/alire/alire-builds.adb | 135 +++++++++++++++ src/alire/alire-builds.ads | 42 +++++ src/alire/alire-config-edit.adb | 14 +- src/alire/alire-config-edit.ads | 23 ++- src/alire/alire-config.ads | 7 +- src/alire/alire-crate_configuration.adb | 4 +- src/alire/alire-directories.adb | 34 ++++ src/alire/alire-directories.ads | 49 +++++- src/alire/alire-origins.ads | 5 + src/alire/alire-paths-vault.ads | 16 ++ src/alire/alire-paths.ads | 5 + src/alire/alire-platforms-current.ads | 2 + src/alire/alire-releases.adb | 6 +- src/alire/alire-roots.adb | 161 +++++++++++------- src/alire/alire-roots.ads | 33 +++- src/alire/alire-shared.adb | 39 +---- src/alire/alire-shared.ads | 14 +- src/alire/alire-toolchains.ads | 3 + src/alire/alire-utils-tools.adb | 17 +- src/alire/alire-utils-tools.ads | 10 +- src/alr/alr-commands-version.adb | 33 ++-- src/alr/alr-commands.adb | 2 - testsuite/Dockerfile | 1 + .../config/shared-deps-profiles/test.py | 0 .../config/shared-deps-profiles/test.yaml | 0 testsuite/drivers/alr.py | 26 +++ testsuite/drivers/asserts.py | 8 +- testsuite/drivers/driver/docker_wrapper.py | 1 - testsuite/drivers/helpers.py | 2 +- .../build_profile/alr_build_switches/test.py | 4 +- testsuite/tests/config/shared-deps/test.py | 66 +++---- .../dockerized/misc/default-cache/test.py | 28 ++- .../get/external-tool-dependency/test.py | 2 + testsuite/tests/get/git-local/test.py | 1 + testsuite/tests/get/unpack-in-place/test.py | 1 + .../tests/printenv/compiler-indirect/test.py | 2 +- testsuite/tests/toolchain/directories/test.py | 2 +- .../tests/workflows/actions-as-root/test.py | 1 + 39 files changed, 623 insertions(+), 205 deletions(-) create mode 100644 src/alire/alire-builds.adb create mode 100644 src/alire/alire-builds.ads create mode 100644 src/alire/alire-paths-vault.ads rename testsuite/{tests => deprecated}/config/shared-deps-profiles/test.py (100%) rename testsuite/{tests => deprecated}/config/shared-deps-profiles/test.yaml (100%) diff --git a/doc/user-changes.md b/doc/user-changes.md index 82f557e6..c3e57fe6 100644 --- a/doc/user-changes.md +++ b/doc/user-changes.md @@ -6,6 +6,35 @@ stay on top of `alr` new features. ## Release `2.0-dev` +### Deprecation of `dependencies.dir` in favor of `dependencies.shared` + +PR [#1419](https://github.com/alire-project/alire/pull/1419) + +A new system of shared sources and builds is being implemented, which will +ensure full consistency and reuse of builds. + +In the new system (still disabled; enable it by setting `alr config --set +dependencies.shared true`), dependencies will no longer be stored under +`/alire/cache/dependencies`. Instead, three new directories are +introduced: + +- `$HOME/.cache/alire/releases`: contains sources for read-only purposes and + binary releases (except toolchains, see below). +- `$HOME/.cache/alire/builds`: contains source builds for a unique combination + of build profile, GPR externals and environment variables. +- `$HOME/.cache/alire/toolchains`: contains downloaded toolchains. + +The previous `$HOME/.cache/alire/dependencies` that contained both toolchains +and binary releases is no longer in use. + +Users wanting to modify dependencies in tandem within a workspace are +encouraged to use the pin system. + +If these default locations for shared dependencies must be relocated, this can +be achieved by using a new configuration path (`ALR_CONFIG` or `-c` global +switch). In that case, the aforementioned paths will be found under +`/path/to/config/cache`. + ### Request review of an index submission with `alr publish --request-review` PR [#1409](https://github.com/alire-project/alire/pull/1409) diff --git a/src/alire/alire-builds.adb b/src/alire/alire-builds.adb new file mode 100644 index 00000000..8dad9bc4 --- /dev/null +++ b/src/alire/alire-builds.adb @@ -0,0 +1,135 @@ +with AAA.Strings; + +with Alire.Config.Edit; +with Alire.Directories; +with Alire.OS_Lib.Subprocess; +with Alire.Paths.Vault; +with Alire.Platforms.Current; +with Alire.Properties.Actions.Executor; +with Alire.Utils.Tools; + +package body Alire.Builds is + + use Directories.Operators; -- "/" + + ---------------------------- + -- Sandboxed_Dependencies -- + ---------------------------- + + function Sandboxed_Dependencies return Boolean + is (not Config.DB.Get (Config.Keys.Dependencies_Shared, + Config.Defaults.Dependencies_Shared)); + + ------------------- + -- To_Msys2_Path -- + ------------------- + -- Convert C:\blah\blah into /c/blah/blah. This is needed because otherwise + -- rsync confuses drive letters with remote hostnames. This might be useful + -- in our troubles with tar? + function To_Msys2_Path (Path : Absolute_Path) return String + is + begin + if not Platforms.Current.On_Windows then + return Path; + end if; + + declare + use AAA.Strings; + New_Path : String (1 .. Path'Length) := Path; + begin + -- Replace ':' with '/' + New_Path (2) := '/'; + + -- Replace '\' with '/' + return "/" & Replace (New_Path, "\", "/"); + end; + end To_Msys2_Path; + + ---------- + -- Sync -- + ---------- + + procedure Sync (Release : Releases.Release; + Was_There : out Boolean) + is + Src : constant Absolute_Path := Paths.Vault.Path + / Release.Deployment_Folder; + Dst : constant Absolute_Path := Builds.Path (Release); + Completed : Directories.Completion := Directories.New_Completion (Dst); + + use AAA.Strings; + begin + Was_There := False; + + if Completed.Is_Complete then + Trace.Detail ("Skipping build syncing to existing " & Dst); + Was_There := True; + return; + end if; + + Directories.Create_Tree (Dst); + -- Ensure the sync destination dir exists + + Utils.Tools.Check_Tool (Utils.Tools.Rsync); + + declare + Busy : constant Simple_Logging.Ongoing := + Simple_Logging.Activity + ("Syncing " & Release.Milestone.TTY_Image) + with Unreferenced; + begin + OS_Lib.Subprocess.Checked_Spawn + (Command => "rsync", + Arguments => + To_Vector ("--del") + & "-aChH" + -- Archive, no CVS folders, keep hard links, human units + & (if Log_Level > Detail then To_Vector ("-P") else Empty_Vector) + & (if Log_Level < Info then To_Vector ("-q") else Empty_Vector) + -- Trailing '/' to access contents directly in the following paths + & To_Msys2_Path (Src / "") + & To_Msys2_Path (Dst / "") + ); + -- TODO: this may take some time, and rsync doesn't have a way + -- to show oneliner progress, so at some point we may want to use + -- something like GNAT.Expect for our spawns, and spin on newlines + -- for example. + end; + + declare + use Directories; + Work_Dir : Guard (Enter (Dst)) with Unreferenced; + begin + Alire.Properties.Actions.Executor.Execute_Actions + (Release => Release, + Env => Platforms.Current.Properties, + Moment => Properties.Actions.Post_Fetch); + exception + when E : others => + Log_Exception (E); + Trace.Warning ("A post-fetch action failed, " & + "re-run with -vv -d for details"); + end; + + Completed.Mark (Complete => True); + end Sync; + + ---------- + -- Path -- + ---------- + + function Path return Absolute_Path + is (Config.Edit.Cache_Path + / Paths.Build_Folder_Inside_Working_Folder); + + ---------- + -- Path -- + ---------- + + function Path (Release : Releases.Release) return Absolute_Path + is (Builds.Path + / (Release.Deployment_Folder + & "_deadbeef")); + -- TODO: implement actual hashing of environment for a release + +end Alire.Builds; diff --git a/src/alire/alire-builds.ads b/src/alire/alire-builds.ads new file mode 100644 index 00000000..4069a38e --- /dev/null +++ b/src/alire/alire-builds.ads @@ -0,0 +1,42 @@ +with Alire.Releases; + +package Alire.Builds is + + -- Stuff for shared builds: build hashes, syncing from the vault, etc. + + -- The new "shared builds" mechanism aims to provide safe sharing of + -- builds, which should result in decreased recompilations (as most + -- configs should be compiled only once, unless some actions are touching + -- something) and decreased disk use (as multiple workspaces will reuse the + -- same builds). + + -- This relies on having a separate source folder for each build + -- configuration (a 'build folder'). This build folder is uniquely + -- identified by a hash derived from the configuration variables, + -- environment variables, GPR externals, and build profile that affect a + -- release, according to manifests in scope. + + -- The build folder is created on-demand, under the name of + -- , syncing it from a "read-only" + -- location, the 'vault', where all releases are fetched initially. + + -- To be able to quickly identify available toolchains without maintaining + -- a "state" file, these are now separately stored, whereas in the past + -- they were stored together with all binary releases. Since now we'll have + -- many more shared releases in the vault, finding toolchains could take + -- much more time, hence the separate storage. + + function Sandboxed_Dependencies return Boolean; + -- Queries config to see if dependencies should be sandboxed in workspace + + procedure Sync (Release : Releases.Release; + Was_There : out Boolean) + with Pre => Release.Origin.Requires_Build; + + function Path return Absolute_Path; + -- Location of shared builds + + function Path (Release : Releases.Release) return Absolute_Path; + -- Computes the complete path in which the release is going to be built + +end Alire.Builds; diff --git a/src/alire/alire-config-edit.adb b/src/alire/alire-config-edit.adb index 9f50e3a9..b40517f2 100644 --- a/src/alire/alire-config-edit.adb +++ b/src/alire/alire-config-edit.adb @@ -1,6 +1,7 @@ with Ada.Text_IO; with Alire.Environment; +with Alire.Paths; with Alire.Platforms.Folders; with Alire.Platforms.Current; with Alire.Utils; @@ -181,6 +182,8 @@ package body Alire.Config.Edit is end Load_Config; + Default_Config_Path : constant Absolute_Path := Platforms.Folders.Config; + ---------- -- Path -- ---------- @@ -191,10 +194,19 @@ package body Alire.Config.Edit is return Config_Path.all; else return OS_Lib.Getenv (Environment.Config, - Platforms.Folders.Config); + Default_Config_Path); end if; end Path; + ---------------- + -- Cache_Path -- + ---------------- + + function Cache_Path return Absolute_Path + is (if Path = Default_Config_Path + then Platforms.Folders.Cache + else Path / Paths.Cache_Folder_Inside_Working_Folder); + -------------- -- Set_Path -- -------------- diff --git a/src/alire/alire-config-edit.ads b/src/alire/alire-config-edit.ads index 011bb873..4780579c 100644 --- a/src/alire/alire-config-edit.ads +++ b/src/alire/alire-config-edit.ads @@ -1,7 +1,6 @@ with AAA.Strings; with Alire.Directories; -with Alire.Paths; with CLIC.Config; @@ -47,6 +46,12 @@ package Alire.Config.Edit is -- * An ALR_CONFIG env given folder -- * Default per-platform path (see alire-platforms-*) + function Cache_Path return Absolute_Path; + -- The location for data that will be recreated if missing; defaults to + -- Platforms.Folders.Cache; if Path above is overridden, the cache will + -- be inside the config folder so as to keep that configuration completely + -- isolated. + procedure Set_Path (Path : Absolute_Path); -- Override global config folder path @@ -103,17 +108,11 @@ private Builtins : constant array (Natural range <>) of Builtin_Entry := ( - (+Keys.Dependencies_Dir, - Cfg_Existing_Absolute_Path, - +("Overrides the default storage directory of regular (non-binary) " - & " dependencies. When unset, releases are stored inside each " - & "workspace at '" & TTY.URL - (Paths.Working_Folder_Inside_Root - / Paths.Cache_Folder_Inside_Working_Folder - / Paths.Deps_Folder_Inside_Cache_Folder) & "'. " - & "Sharing dependencies across workspaces may save disk space, but " - & "it is generally not recommended as different dependents may need " - & "to configure dependencies differently. Use at your own risk." + (+Keys.Dependencies_Shared, + Cfg_Bool, + +("When true, dependencies are downloaded and built in a shared " + & "location inside the global cache. When false (default), " + & "dependencies are sandboxed in each workspace." )), (+Keys.Index_Auto_Community, diff --git a/src/alire/alire-config.ads b/src/alire/alire-config.ads index b650e3ad..e5def09f 100644 --- a/src/alire/alire-config.ads +++ b/src/alire/alire-config.ads @@ -23,7 +23,7 @@ package Alire.Config with Preelaborate is -- A few predefined keys that are used in several places. This list is -- not exhaustive. - Dependencies_Dir : constant Config_Key := "dependencies.dir"; + Dependencies_Shared : constant Config_Key := "dependencies.shared"; Editor_Cmd : constant Config_Key := "editor.cmd"; @@ -88,12 +88,15 @@ package Alire.Config with Preelaborate is package Defaults is - Warning_Old_Index : constant Boolean := True; + Dependencies_Shared : constant Boolean := False; + -- TODO: enable it when hashing is complete Index_Host : constant String := "https://github.com"; Index_Owner : constant String := "alire-project"; Index_Repo_Name : constant String := "alire-index"; + Warning_Old_Index : constant Boolean := True; + end Defaults; end Alire.Config; diff --git a/src/alire/alire-crate_configuration.adb b/src/alire/alire-crate_configuration.adb index d1c376f4..db26e3b3 100644 --- a/src/alire/alire-crate_configuration.adb +++ b/src/alire/alire-crate_configuration.adb @@ -551,9 +551,7 @@ package body Alire.Crate_Configuration is is use type Profile_Maps.Map; begin - return - Config.DB.Get (Config.Keys.Dependencies_Dir, "") /= "" or else - This.Profile_Map /= Last_Build_Profiles; + return This.Profile_Map /= Last_Build_Profiles; end Must_Regenerate; --------------------------- diff --git a/src/alire/alire-directories.adb b/src/alire/alire-directories.adb index 0b713353..590ca24c 100644 --- a/src/alire/alire-directories.adb +++ b/src/alire/alire-directories.adb @@ -1065,4 +1065,38 @@ package body Alire.Directories is end if; end Touch; + ---------- + -- File -- + ---------- + + function File (This : Completion) return Absolute_File + is (This.Path + / Paths.Working_Folder_Inside_Root + / Paths.Complete_Flag); + + ----------------- + -- Is_Complete -- + ----------------- + + function Is_Complete (This : Completion) return Boolean + is (Is_File (This.File)); + + ---------- + -- Mark -- + ---------- + + procedure Mark (This : in out Completion; + Complete : Boolean) + is + begin + if Complete then + Create_Tree (Parent (This.File)); + Touch (This.File); + else + if This.Is_Complete then + Delete_Tree (This.File); + end if; + end if; + end Mark; + end Alire.Directories; diff --git a/src/alire/alire-directories.ads b/src/alire/alire-directories.ads index c2ebf8c9..227173b0 100644 --- a/src/alire/alire-directories.ads +++ b/src/alire/alire-directories.ads @@ -55,6 +55,7 @@ package Alire.Directories is -- Calls Ensure_Deletable and then uses GNATCOLL.VFS deletion procedure Delete_Tree (Path : Any_Path) renames Force_Delete; + -- Delete Path, and anythin below if it was a dir function Find_Files_Under (Folder : String; Name : String; @@ -102,9 +103,10 @@ package Alire.Directories is -- Remove softlinks only (not their targets) at Path and subdirs when -- Recursive. - procedure Touch (File : File_Path); - -- If the file exists, update last edition time; otherwise create it. If - -- File denotes anything else than a regular file, raise. + procedure Touch (File : File_Path) + with Pre => Is_Directory (Parent (File)); + -- If the file exists, update last edition time; otherwise create it. + -- If File denotes anything else than a regular file, raise. Traverse_Tree_Prune_Dir : exception; -- In Traverse_Tree, the Doing procedure can raise this exception to @@ -220,6 +222,26 @@ package Alire.Directories is -- called, on going out of scope the Replacer will remove the temporary and -- the original file remains untouched. + -- To ensure that certain download/copy/sync operations are complete, we + -- use this type that will check/delete/create a /alire/complete_copy + -- canary file. + + type Completion (<>) is tagged limited private; + + function New_Completion (Path : Directory_Path) return Completion; + -- This is the destination folder whose operation we want to ensure was + -- completed: a download/copy destination, for example. + + function Is_Complete (This : Completion) return Boolean; + -- Say if the operation at This path was already complete in which case + -- nothing should be done. + + procedure Mark (This : in out Completion; + Complete : Boolean) + with Post => This.Is_Complete = Complete; + -- Set/remove the canary flag of the operation on path being complete. This + -- should be called when the operation has actually been completed. + private ------------ @@ -267,4 +289,25 @@ private function Find_Relative_Path_To (Path : Any_Path) return Any_Path is (Find_Relative_Path (Current, Path)); + ---------------- + -- Completion -- + ---------------- + + type Completion (Length : Natural) is + new Ada.Finalization.Limited_Controlled with + record + Path : Absolute_Path (1 .. Length); + end record; + + function File (This : Completion) return Absolute_File; + + -------------------- + -- New_Completion -- + -------------------- + + function New_Completion (Path : Directory_Path) return Completion + is (Ada.Finalization.Limited_Controlled with + Length => Full_Name (Path)'Length, + Path => Full_Name (Path)); + end Alire.Directories; diff --git a/src/alire/alire-origins.ads b/src/alire/alire-origins.ads index 233fbf83..4ba6664d 100644 --- a/src/alire/alire-origins.ads +++ b/src/alire/alire-origins.ads @@ -113,6 +113,11 @@ package Alire.Origins is -- an index, instead of coming from locally detected executables or system -- packages. + function Requires_Build (This : Origin) return Boolean + is (case This.Kind is + when Binary_Archive | External | System => False, + when Source_Archive | VCS_Kinds | Filesystem => True); + function Short_Unique_Id (This : Origin) return String with Pre => This.Kind in Git | Hg | Archive_Kinds; diff --git a/src/alire/alire-paths-vault.ads b/src/alire/alire-paths-vault.ads new file mode 100644 index 00000000..23d4f317 --- /dev/null +++ b/src/alire/alire-paths-vault.ads @@ -0,0 +1,16 @@ +with Alire.Config.Edit; + +package Alire.Paths.Vault is + + -- We call the 'vault' (so not to abuse "cache" which has another meaning) + -- the "read-only" location where all releases are initially deployed + -- as-is, without actions being run (unless for binary releases, which are + -- deployed once and have their actions run at that time). Before building, + -- a non-binary release is synced to the actual build location and actions + -- are run there (see Alire.Builds). + + function Path return Absolute_Path + is (Config.Edit.Cache_Path + / Release_Folder_Inside_Working_Folder); + +end Alire.Paths.Vault; diff --git a/src/alire/alire-paths.ads b/src/alire/alire-paths.ads index e9a8b96d..5295c595 100644 --- a/src/alire/alire-paths.ads +++ b/src/alire/alire-paths.ads @@ -5,6 +5,8 @@ package Alire.Paths with Preelaborate is Crate_File_Name : constant String := "alire.toml"; -- Name of the manifest file in a regular workspace + Build_Folder_Inside_Working_Folder : constant Relative_Path := "builds"; + Cache_Folder_Inside_Working_Folder : constant Relative_Path := "cache"; Deps_Folder_Inside_Cache_Folder : constant Relative_Path := "dependencies"; @@ -20,6 +22,9 @@ package Alire.Paths with Preelaborate is Scripts_Graph_Easy : constant String := "graph-easy"; -- Script for ASCII graphs + Complete_Flag : constant String := "complete_copy"; + -- We use /alire/ to mark that a file operation was completed + private Crate_File_Extension_With_Dot : constant String := ".toml"; diff --git a/src/alire/alire-platforms-current.ads b/src/alire/alire-platforms-current.ads index 5136598b..8028ad70 100644 --- a/src/alire/alire-platforms-current.ads +++ b/src/alire/alire-platforms-current.ads @@ -35,6 +35,8 @@ package Alire.Platforms.Current is -------------------------------- -- Beyond this point, nothing has to be done in the body + function On_Windows return Boolean is (Operating_System in Windows); + Disable_Distribution_Detection : Boolean := False with Atomic; function Distribution return Platforms.Distributions; diff --git a/src/alire/alire-releases.adb b/src/alire/alire-releases.adb index 19a83c89..f7440cae 100644 --- a/src/alire/alire-releases.adb +++ b/src/alire/alire-releases.adb @@ -252,6 +252,8 @@ package body Alire.Releases is use Alire.Directories; use all type Alire.Properties.Actions.Moments; Folder : constant Any_Path := Parent_Folder / This.Deployment_Folder; + Completed : Directories.Completion := + Directories.New_Completion (Folder); ------------------------------ -- Backup_Upstream_Manifest -- @@ -300,7 +302,7 @@ package body Alire.Releases is -- Deploy if the target dir is not already there - if Ada.Directories.Exists (Folder) then + if Completed.Is_Complete then Was_There := True; Trace.Detail ("Skipping checkout of already available " & This.Milestone.Image); @@ -348,6 +350,8 @@ package body Alire.Releases is end; end if; + Completed.Mark (Complete => True); + exception when E : others => -- Clean up if deployment failed after the initial deployment (e.g. diff --git a/src/alire/alire-roots.adb b/src/alire/alire-roots.adb index 0a0bdebb..8eca2319 100644 --- a/src/alire/alire-roots.adb +++ b/src/alire/alire-roots.adb @@ -1,5 +1,6 @@ with Ada.Unchecked_Deallocation; +with Alire.Builds; with Alire.Conditional; with Alire.Config; with Alire.Dependencies.Containers; @@ -10,11 +11,13 @@ with Alire.Install; with Alire.Manifest; with Alire.Origins; with Alire.OS_Lib; +with Alire.Paths.Vault; with Alire.Properties.Actions.Executor; with Alire.Roots.Optional; with Alire.Shared; with Alire.Solutions.Diffs; with Alire.Spawn; +with Alire.Toolchains; with Alire.User_Pins.Maps; with Alire.Utils.TTY; with Alire.Utils.User_Input; @@ -192,15 +195,18 @@ package body Alire.Roots is This.Set_Build_Profiles (Crate_Configuration.Last_Build_Profiles); end if; - -- Check if crate configuration should be re-generated + -- Check if crate configuration should be re-generated. This is the old + -- behavior; for shared builds, config needs to be generated only once. This.Load_Configuration; - if This.Configuration.Must_Regenerate then + if Builds.Sandboxed_Dependencies + and then This.Configuration.Must_Regenerate + then This.Generate_Configuration; end if; This.Configuration.Ensure_Complete; - -- For building the configuration must be complete + -- For proceeding to build, the configuration must be complete if Export_Build_Env then This.Export_Build_Environment; @@ -619,13 +625,6 @@ package body Alire.Roots is procedure Deploy_Dependencies (This : in out Roots.Root) is - ----------------------------- - -- Dependencies_Are_Shared -- - ----------------------------- - - function Dependencies_Are_Shared return Boolean - is (Config.DB.Get (Config.Keys.Dependencies_Dir, "") /= ""); - -------------------- -- Deploy_Release -- -------------------- @@ -691,34 +690,44 @@ package body Alire.Roots is declare Rel : constant Releases.Release := Dep.Release; begin - if Rel.Origin.Kind in Origins.Binary_Archive then - - -- Binary releases are always installed as shared releases - Shared.Share (Rel); + Trace.Debug ("deploy: process " & Rel.Milestone.TTY_Image); - elsif Dep.Is_Shared and then not Rel.Origin.Is_Index_Provided then + if Toolchains.Is_Tool (Rel) then - -- Externals shouldn't leave a trace in the binary cache - Trace.Debug ("deploy: skip shared external"); + -- Toolchain crates are installed to their own place + Shared.Share (Rel); else - -- Remaining cases expect to receive a Deploy call, even - -- externals in the working directory - Rel.Deploy (Env => This.Environment, - Parent_Folder => - This.Dependencies_Dir (Rel.Name), - Perform_Actions => False, - Was_There => Was_There, - Create_Manifest => - Dep.Is_Shared or else Dependencies_Are_Shared, - Include_Origin => - Dep.Is_Shared); - - -- Always run the post-fetch on update of dependencies, in - -- case there is some interaction with some other updated - -- dependency, even for crates that didn't change. - Run_Post_Fetch (Rel); + -- Remaining cases need deploying and running of actions + + Rel.Deploy + (Env => This.Environment, + Parent_Folder => This.Release_Parent (Rel, For_Deploy), + Was_There => Was_There, + Perform_Actions => + not This.Requires_Build_Sync (Rel), + -- In sandbox mode, this is the final destination so + -- post-fetch must run. For binaries not built, that + -- always live in the vault, we too run post-fetch + -- immediately as this is the only chance. + Create_Manifest => + not Builds.Sandboxed_Dependencies, + -- Merely for back-compatibility + Include_Origin => + not Builds.Sandboxed_Dependencies + -- Merely for back-compatibility + ); + + -- Sync sources to its shared build location + + if not Builds.Sandboxed_Dependencies then + Builds.Sync (Rel, Was_There); + end if; + + -- At this point, post-fetch have been run by either + -- Builds.Sync or Rel.Deploy; also completion will only + -- have succeeded if the post-fetch actions have too. -- If the release was newly deployed, we can inform about its -- nested crates now. @@ -751,7 +760,11 @@ package body Alire.Roots is -- Update/Create configuration files + This.Load_Configuration; This.Generate_Configuration; + -- TODO: this should be made more granular to only generate + -- configurations of newly synced build sources, since with the + -- new shared builds system configs do not change once created. -- Check that the solution does not contain suspicious dependencies, -- taking advantage that this procedure is called whenever a change @@ -1260,30 +1273,41 @@ package body Alire.Roots is use OS_Lib; - ---------------------- - -- Dependencies_Dir -- - ---------------------- + ------------------------- + -- Requires_Build_Sync -- + ------------------------- - function Dependencies_Dir (This : in out Root; - Crate : Crate_Name) - return Any_Path + function Requires_Build_Sync (This : in out Root; + Rel : Releases.Release) + return Boolean + is (Rel.Origin.Requires_Build + and then not Builds.Sandboxed_Dependencies + and then not This.Solution.State (Rel).Is_Linked); + + -------------------- + -- Release_Parent -- + -------------------- + + function Release_Parent (This : in out Root; + Rel : Releases.Release; + Usage : Usages) + return Absolute_Path is begin - if This.Solution.State (Crate).Is_Solved then - if This.Solution.State (Crate).Is_Shared then - return Shared.Path; - elsif Config.DB.Get (Config.Keys.Dependencies_Dir, "") /= "" then - return Config.DB.Get (Config.Keys.Dependencies_Dir, ""); - else - return - This.Cache_Dir - / Paths.Deps_Folder_Inside_Cache_Folder; - end if; + if Toolchains.Is_Tool (Rel) then + return Shared.Path; + elsif Builds.Sandboxed_Dependencies then + -- Note that, even for releases not requiring a build (e.g. + -- externals), in sandboxed mode we are creating a folder for them + -- in the workspace cache in which actions could be run. + return This.Dependencies_Dir; else - raise Program_Error - with "deploy base only applies to solved releases"; + case Usage is + when For_Deploy => return Paths.Vault.Path; + when For_Build => return Builds.Path; + end case; end if; - end Dependencies_Dir; + end Release_Parent; ------------------ -- Release_Base -- @@ -1291,7 +1315,7 @@ package body Alire.Roots is function Release_Base (This : in out Root; Crate : Crate_Name) - return Any_Path + return Absolute_Path is begin if This.Release.Element.Name = Crate then @@ -1300,7 +1324,11 @@ package body Alire.Roots is declare Rel : constant Releases.Release := Release (This, Crate); begin - return This.Dependencies_Dir (Crate) / Rel.Base_Folder; + if Builds.Sandboxed_Dependencies then + return This.Release_Parent (Rel, For_Build) / Rel.Base_Folder; + else + return Builds.Path (Rel); + end if; end; elsif This.Solution.State (Crate).Is_Linked then return This.Solution.State (Crate).Link.Path; @@ -1367,6 +1395,15 @@ package body Alire.Roots is function Cache_Dir (This : Root) return Absolute_Path is (This.Working_Folder / Paths.Cache_Folder_Inside_Working_Folder); + ---------------------- + -- Dependencies_Dir -- + ---------------------- + + function Dependencies_Dir (This : in out Root) return Any_Path + is (if Builds.Sandboxed_Dependencies + then This.Cache_Dir / Paths.Deps_Folder_Inside_Cache_Folder + else Paths.Vault.Path); + -------------- -- Pins_Dir -- -------------- @@ -1464,8 +1501,8 @@ package body Alire.Roots is -- a non-exhaustive sync of pins, that will anyway detect evident -- changes (new/removed pins, changed explicit commits). - This.Sync_Dependencies (Silent => Silent, - Interact => Interact); + This.Update_Dependencies (Silent => Silent, + Interact => Interact); -- Don't ask for confirmation as this is an automatic update in -- reaction to a manually edited manifest, and we need the lockfile -- to match the manifest. As any change in dependencies will be @@ -1537,7 +1574,7 @@ package body Alire.Roots is -- And look for updates in dependencies - This.Sync_Dependencies + This.Update_Dependencies (Allowed => Allowed, Silent => Silent, Interact => Interact and not CLIC.User_Input.Not_Interactive); @@ -1586,11 +1623,11 @@ package body Alire.Roots is Options => Options); end Compute_Update; - ----------------------- - -- Sync_Dependencies -- - ----------------------- + ------------------------- + -- Update_Dependencies -- + ------------------------- - procedure Sync_Dependencies + procedure Update_Dependencies (This : in out Root; Silent : Boolean; -- Do not output anything Interact : Boolean; -- Request confirmation from the user @@ -1675,7 +1712,7 @@ package body Alire.Roots is Trace.Detail ("Update completed"); end; - end Sync_Dependencies; + end Update_Dependencies; -------------------- -- Temporary_Copy -- diff --git a/src/alire/alire-roots.ads b/src/alire/alire-roots.ads index 663138c2..e4424d02 100644 --- a/src/alire/alire-roots.ads +++ b/src/alire/alire-roots.ads @@ -117,11 +117,27 @@ package Alire.Roots is Post => Release'Result.Provides (Crate); -- Retrieve a release, that can be either the root or any in the solution + type Usages is (For_Deploy, For_Build); + + function Release_Parent (This : in out Root; + Rel : Releases.Release; + Usage : Usages) + return Absolute_Path + with Pre => This.Solution.Contains_Release (Rel.Name); + -- The dir into which a release is deployed taking into account all config + -- and release particulars (binary...) + function Release_Base (This : in out Root; Crate : Crate_Name) - return Any_Path; + return Absolute_Path; -- Find the base folder in which a release can be found for the given root + function Requires_Build_Sync (This : in out Root; + Rel : Releases.Release) + return Boolean + with Pre => This.Solution.Contains_Release (Rel.Name); + -- Says if the release requires a build copy taking into account everything + function Nonabstract_Crates (This : in out Root) return Containers.Crate_Name_Sets.Set; -- Return names of crates in the solution that have a buildable release, @@ -181,7 +197,7 @@ package Alire.Roots is procedure Deploy_Dependencies (This : in out Root); -- Download all dependencies not already on disk from This.Solution - procedure Sync_Dependencies + procedure Update_Dependencies (This : in out Root; Silent : Boolean; -- Do not output anything Interact : Boolean; -- Request confirmation from the user @@ -285,6 +301,12 @@ package Alire.Roots is function Cache_Dir (This : Root) return Absolute_Path; -- The "alire/cache" dir inside the root path, containing releases and pins + function Dependencies_Dir (This : in out Root) return Any_Path; + -- The path at which dependencies are deployed, which will + -- be either Paths.Vault.Path if dependencies are shared, or + -- /alire/cache/dependencies when dependencies are + -- sandboxed (legacy pre-2.0 mode). + function Crate_File (This : Root) return Absolute_Path; -- The "/path/to/alire.toml" file inside Working_Folder @@ -373,11 +395,4 @@ private -- Renames the manifest and lockfile to their regular places, making this -- root a regular one to all effects. - function Dependencies_Dir (This : in out Root; - Crate : Crate_Name) - return Any_Path; - -- The path at which dependencies have to be deployed, which for regular - -- releases is simply ./alire/cache/dependencies, unless overridden by the - -- config option `dependencies.dir` - end Alire.Roots; diff --git a/src/alire/alire-shared.adb b/src/alire/alire-shared.adb index c8285ab3..37167dda 100644 --- a/src/alire/alire-shared.adb +++ b/src/alire/alire-shared.adb @@ -1,15 +1,12 @@ with Ada.Directories; -with Alire.Config; +with Alire.Config.Edit; with Alire.Containers; with Alire.Directories; -with Alire.Environment; with Alire.Index; with Alire.Manifest; with Alire.Origins; -with Alire.OS_Lib; with Alire.Paths; -with Alire.Platforms.Folders; with Alire.Properties.Actions; with Alire.Root; with Alire.Toolchains.Solutions; @@ -87,36 +84,12 @@ package body Alire.Shared is return Result; end Available; - Global_Cache_Path : access String; - ---------- -- Path -- ---------- function Path return String - is ((if Global_Cache_Path /= null - then Global_Cache_Path.all - else OS_Lib.Getenv (Environment.Config, - Platforms.Folders.Cache)) - -- Up to here, it's the default prefix or an overriden prefix - / - (if Global_Cache_Path = null and then - OS_Lib.Getenv (Environment.Config, "") = "" - then Paths.Deps_Folder_Inside_Cache_Folder - else Paths.Cache_Folder_Inside_Working_Folder - / Paths.Deps_Folder_Inside_Cache_Folder) - -- This second part is either cache/dependencies or just dependencies, - -- depending on if the location is shared with the config folder or not - ); - - -------------- - -- Set_Path -- - -------------- - - procedure Set_Path (Path : Absolute_Path) is - begin - Global_Cache_Path := new String'(Path); - end Set_Path; + is (Config.Edit.Cache_Path / "toolchains"); ----------- -- Share -- @@ -165,14 +138,6 @@ package body Alire.Shared is begin - -- See if it is a valid installable origin - - if Release.Origin.Kind in Origins.External_Kinds then - Raise_Checked_Error - ("Only regular releases can be installed, but the requested release" - & " has origin of kind " & Release.Origin.Kind'Image); - end if; - if not Is_Installable then Recoverable_Error ("Releases with both dependencies and post-fetch actions are not " diff --git a/src/alire/alire-shared.ads b/src/alire/alire-shared.ads index bcb5fdbb..aabdba75 100644 --- a/src/alire/alire-shared.ads +++ b/src/alire/alire-shared.ads @@ -6,8 +6,9 @@ with CLIC.User_Input; package Alire.Shared is - -- Stuff about shared/binary crates that are deployed not in the local - -- workspace but in the shared configuration folder. + -- Since the new shared builds infrastructure, this applies exclusively + -- to releases that belong to a toolchain. TODO: migrate it to either + -- Alire.Toolchains or Alire.Toolchains.Shared. function Available (Detect_Externals : Boolean := True) return Releases.Containers.Release_Set; @@ -20,13 +21,8 @@ package Alire.Shared is -- Constraint_Error if not among Available. function Path return Any_Path; - -- Returns the base folder in which all shared releases live: - -- * /cache/dependencies if set with --config/-c - -- * /cache/dependencies if set through ALR_CONFIG - -- * ~/.cache/alire/dependencies by default - - procedure Set_Path (Path : Absolute_Path); - -- Override the location of the global cache location + -- Returns the base folder in which all shared releases live, defaults to + -- /toolchains procedure Share (Release : Releases.Release; Location : Any_Path := Path); diff --git a/src/alire/alire-toolchains.ads b/src/alire/alire-toolchains.ads index 06d0ed71..32ae9bcd 100644 --- a/src/alire/alire-toolchains.ads +++ b/src/alire/alire-toolchains.ads @@ -22,6 +22,9 @@ package Alire.Toolchains is .Union (Name_Sets.To_Set (GPRbuild_Crate)); -- All crates that are part of the provided binary toolchain + function Is_Tool (Release : Releases.Release) return Boolean + is (for some Tool of Tools => Release.Provides (Tool)); + function Any_Tool (Crate : Crate_Name) return Dependencies.Dependency; -- Returns a dependency on crate* diff --git a/src/alire/alire-utils-tools.adb b/src/alire/alire-utils-tools.adb index 9d3dc4e9..93c76560 100644 --- a/src/alire/alire-utils-tools.adb +++ b/src/alire/alire-utils-tools.adb @@ -40,6 +40,7 @@ package body Alire.Utils.Tools is when Unzip => "unzip", when Curl => "curl", when Mercurial => "hg", + when Rsync => "rsync", when Subversion => "svn"); ----------------------------- @@ -60,15 +61,13 @@ package body Alire.Utils.Tools is | Homebrew | Macports => return (case Tool is when Easy_Graph => - (if Distribution = Centos or else - Distribution = Fedora or else - Distribution = Rhel or else - Distribution = Suse - then "perl-Graph-Easy" - elsif Distribution /= Msys2 and Distribution /= Arch - then "libgraph-easy-perl" - else ""), - when Git | Tar | Unzip | Curl => Exec_For_Tool (Tool), + (if Distribution in Centos | Fedora | Rhel | Suse + then "perl-Graph-Easy" + elsif Distribution /= Msys2 and Distribution /= Arch + then "libgraph-easy-perl" + else ""), + when Git | Tar | Unzip | Curl | Rsync => + Exec_For_Tool (Tool), when Mercurial => "mercurial", when Subversion => "subversion"); end case; diff --git a/src/alire/alire-utils-tools.ads b/src/alire/alire-utils-tools.ads index 58ccbd3c..cd5da438 100644 --- a/src/alire/alire-utils-tools.ads +++ b/src/alire/alire-utils-tools.ads @@ -1,7 +1,15 @@ package Alire.Utils.Tools is type Tool_Kind is - (Easy_Graph, Git, Tar, Unzip, Curl, Mercurial, Subversion); + (Curl, -- For downloads + Easy_Graph, -- For `--graph` + Git, -- Release origin + Mercurial, -- Release origin + Rsync, -- For shared builds + Subversion, -- Release origin + Tar, -- For source archive origins + Unzip -- For source archive origins + ); function Available (Tool : Tool_Kind) return Boolean; -- Say if tool is already available (attempts detection for the tool, but diff --git a/src/alr/alr-commands-version.adb b/src/alr/alr-commands-version.adb index 8d581856..74999d95 100644 --- a/src/alr/alr-commands-version.adb +++ b/src/alr/alr-commands-version.adb @@ -1,9 +1,10 @@ +with Alire.Builds; with Alire.Config.Edit; with Alire.Directories; with Alire.Index; with Alire.Index_On_Disk.Loading; with Alire.Milestones; -with Alire.Paths; +with Alire.Paths.Vault; with Alire.Properties; with Alire.Roots.Optional; with Alire.Shared; @@ -19,8 +20,12 @@ with CLIC.User_Input; package body Alr.Commands.Version is + use Alire.Directories.Operators; -- "/" + package GNAT_Version is new GNAT.Compiler_Version; + package Paths renames Alire.Paths; + ------------- -- Execute -- ------------- @@ -30,7 +35,6 @@ package body Alr.Commands.Version is Args : AAA.Strings.Vector) is use Alire; - use Alire.Directories.Operators; use all type Alire.Roots.Optional.States; Table : Alire.Utils.Tables.Table; Index_Outcome : Alire.Outcome; @@ -40,8 +44,16 @@ package body Alr.Commands.Version is Root : constant Alire.Roots.Optional.Root := Alire.Roots.Optional.Search_Root (Alire.Directories.Current); - Deps_Dir : constant String := - Config.DB.Get (Alire.Config.Keys.Dependencies_Dir, ""); + Build_Path : constant String := + (if Builds.Sandboxed_Dependencies + then + (if Root.Is_Valid + then Root.Value.Dependencies_Dir + else "" + / Paths.Build_Folder_Inside_Working_Folder + / Paths.Cache_Folder_Inside_Working_Folder + / Paths.Deps_Folder_Inside_Cache_Folder) + else Builds.Path); begin if Args.Count /= 0 then Reportaise_Wrong_Arguments (Cmd.Name & " doesn't take arguments"); @@ -60,14 +72,10 @@ package body Alr.Commands.Version is Table.Append ("").New_Row; Table.Append ("CONFIGURATION").New_Row; Table.Append ("config folder:").Append (Alire.Config.Edit.Path).New_Row; - Table.Append ("cache folder:").Append (Alire.Shared.Path).New_Row; - Table.Append ("dependencies folder:") - .Append (if Deps_Dir = "" - then "" - / Paths.Working_Folder_Inside_Root - / Paths.Cache_Folder_Inside_Working_Folder - / Paths.Deps_Folder_Inside_Cache_Folder - else Deps_Dir).New_Row; + Table.Append ("cache folder:") + .Append (Alire.Config.Edit.Cache_Path).New_Row; + Table.Append ("vault folder:").Append (Paths.Vault.Path).New_Row; + Table.Append ("build folder:").Append (Build_Path).New_Row; Table.Append ("force flag:").Append (Alire.Force'Image).New_Row; Table.Append ("non-interactive flag:") .Append (CLIC.User_Input.Not_Interactive'Image).New_Row; @@ -86,6 +94,7 @@ package body Alr.Commands.Version is & AAA.Strings.Trim (Index.Priority'Image) & ":") .Append ("(" & Index.Name & ") " & Index.Origin).New_Row; end loop; + Table.Append ("toolchain folder:").Append (Alire.Shared.Path).New_Row; Table.Append ("toolchain assistant:") .Append (if Alire.Toolchains.Assistant_Enabled then "enabled" diff --git a/src/alr/alr-commands.adb b/src/alr/alr-commands.adb index 52767302..87686a26 100644 --- a/src/alr/alr-commands.adb +++ b/src/alr/alr-commands.adb @@ -16,7 +16,6 @@ with Alire.Lockfiles; with Alire.Paths; with Alire.Platforms.Current; with Alire.Root; -with Alire.Shared; with Alire.Solutions; with Alire.Toolchains; @@ -486,7 +485,6 @@ package body Alr.Commands is := Ada.Directories.Full_Name (Command_Line_Config_Path.all); begin Alire.Config.Edit.Set_Path (Config_Path); - Alire.Shared.Set_Path (Config_Path); end; end if; diff --git a/testsuite/Dockerfile b/testsuite/Dockerfile index f2dc818a..ef064d3d 100644 --- a/testsuite/Dockerfile +++ b/testsuite/Dockerfile @@ -5,6 +5,7 @@ FROM alire/gnat:ubuntu-lts RUN useradd -m -s /bin/bash user && \ chown user:user /home/user +RUN apt-get update && apt-get install -y gnat gprbuild rsync RUN pip3 install e3-testsuite WORKDIR /testsuite diff --git a/testsuite/tests/config/shared-deps-profiles/test.py b/testsuite/deprecated/config/shared-deps-profiles/test.py similarity index 100% rename from testsuite/tests/config/shared-deps-profiles/test.py rename to testsuite/deprecated/config/shared-deps-profiles/test.py diff --git a/testsuite/tests/config/shared-deps-profiles/test.yaml b/testsuite/deprecated/config/shared-deps-profiles/test.yaml similarity index 100% rename from testsuite/tests/config/shared-deps-profiles/test.yaml rename to testsuite/deprecated/config/shared-deps-profiles/test.yaml diff --git a/testsuite/drivers/alr.py b/testsuite/drivers/alr.py index f572e522..3e40ff62 100644 --- a/testsuite/drivers/alr.py +++ b/testsuite/drivers/alr.py @@ -60,6 +60,11 @@ def prepare_env(config_dir, env): run_alr("-c", config_dir, "config", "--global", "--set", "warning.old_index", "false") + # Disable shared dependencies (keep old pre-2.0 behavior) not to break lots + # of tests. The post-2.0 behavior will have its own tests. + run_alr("-c", config_dir, "config", "--global", + "--set", "dependencies.shared", "false") + # If distro detection is disabled via environment, configure so in alr if "ALIRE_DISABLE_DISTRO" in env: if env["ALIRE_DISABLE_DISTRO"] == "true": @@ -505,3 +510,24 @@ def alr_publish(name, index_path) return p + + +def alr_config_dir() -> str: + """ + Return the path to the alr configuration directory + """ + return os.environ.get("ALR_CONFIG") + + +def alr_vault_dir() -> str: + """ + Return the path to the vault for release pristine sources + """ + return os.path.join(alr_config_dir(), "cache", "releases") + + +def alr_builds_dir() -> str: + """ + Return the path to the builds directory + """ + return os.path.join(alr_config_dir(), "cache", "builds") \ No newline at end of file diff --git a/testsuite/drivers/asserts.py b/testsuite/drivers/asserts.py index 9515d5b9..e5bc3a02 100644 --- a/testsuite/drivers/asserts.py +++ b/testsuite/drivers/asserts.py @@ -45,11 +45,13 @@ def assert_eq(expected, actual, label=None): def assert_contents(dir: str, expected, regex: str = ""): """ - Check that entries in dir filtered by regex match the list in expected + Check that entries in dir filtered by regex match the list in expected. + Transforms all paths to use forward slashes, and sorts the contents. """ real = contents(dir, regex) - assert real == expected, \ - f"Wanted contents: {expected}\nBut got: {real}\n" + + assert real == sorted([path.replace('\\', '/') for path in expected]), \ + f"Wanted contents:\n{expected}\nBut got:\n{real}\n" def assert_match(expected_re, actual, label=None, flags=re.S): diff --git a/testsuite/drivers/driver/docker_wrapper.py b/testsuite/drivers/driver/docker_wrapper.py index 2cf6558e..e3f013bf 100644 --- a/testsuite/drivers/driver/docker_wrapper.py +++ b/testsuite/drivers/driver/docker_wrapper.py @@ -11,7 +11,6 @@ from e3.testsuite.driver.classic import (ClassicTestDriver, TestSkip) DOCKERFILE = "Dockerfile" -PLACEHOLDER = "TO_BE_COMPUTED" TAG = "alire_testsuite" ENV_FLAG = "ALIRE_DOCKER_ENABLED" # Set to "False" to disable docker tests LABEL_HASH = "hash" diff --git a/testsuite/drivers/helpers.py b/testsuite/drivers/helpers.py index 6a5b83f0..c70bd565 100644 --- a/testsuite/drivers/helpers.py +++ b/testsuite/drivers/helpers.py @@ -45,7 +45,7 @@ def lines_of(filename): # Assert two values are equal or format the differences def compare(found, wanted): - assert found == wanted, 'Got: {}\nWanted: {}'.format(found, wanted) + assert found == wanted, '\nGot:\n{}\nWanted:\n{}'.format(found, wanted) # Check line appears in file diff --git a/testsuite/tests/build_profile/alr_build_switches/test.py b/testsuite/tests/build_profile/alr_build_switches/test.py index 338c5520..855a17ae 100644 --- a/testsuite/tests/build_profile/alr_build_switches/test.py +++ b/testsuite/tests/build_profile/alr_build_switches/test.py @@ -28,14 +28,14 @@ def check_config_changed(): global mtime last_mtime = mtime mtime = os.path.getmtime(bin_config) - assert last_mtime != mtime, "config file did not change" + assert last_mtime != mtime, "config file timestamp did not change" def check_config_not_changed(): global mtime last_mtime = mtime mtime = os.path.getmtime(bin_config) - assert last_mtime == mtime, "config file did not change" + assert last_mtime == mtime, "config file timestamp did unexpectedly change" # Check default profiles for root and dependency diff --git a/testsuite/tests/config/shared-deps/test.py b/testsuite/tests/config/shared-deps/test.py index 1e19ebee..c6ab444e 100644 --- a/testsuite/tests/config/shared-deps/test.py +++ b/testsuite/tests/config/shared-deps/test.py @@ -1,48 +1,54 @@ """ -Check that globally sharing dependencies works as expected +Check that globally sharing builds works as expected """ +import glob import os -from drivers.alr import alr_with, alr_workspace_cache, init_local_crate, run_alr + +from drivers.alr import (alr_builds_dir, alr_vault_dir, alr_with, alr_workspace_cache, + init_local_crate, run_alr) from drivers.asserts import assert_contents, assert_file_exists from drivers.helpers import lines_of -deps_dir = "deps" - -# Check that using a relative path fails, even if it exists -os.mkdir("fake") -run_alr("config", "--global", "--set", "dependencies.dir", "fake", - complain_on_error=False) - -# Check that using an absolute non-existing path fails -run_alr("config", "--global", "--set", "dependencies.dir", - os.path.abspath(deps_dir), - complain_on_error=False) +vault_dir = alr_vault_dir() +build_dir = alr_builds_dir() -# Succeed after creating the destination -os.mkdir(deps_dir) -run_alr("config", "--global", "--set", "dependencies.dir", - os.path.abspath(deps_dir)) +# Enable shared builds +run_alr("config", "--global", "--set", "dependencies.shared", "true") # Create a crate with a dependency init_local_crate() alr_with("hello") -# Ensure the dependencies are where expected -assert_file_exists(os.path.join("..", deps_dir, "hello_1.0.1_filesystem")) -assert_file_exists(os.path.join("..", deps_dir, "libhello_1.0.0_filesystem")) +# Ensure the "read-only" sources are where expected +assert_file_exists(os.path.join(vault_dir, "hello_1.0.1_filesystem")) +assert_file_exists(os.path.join(vault_dir, "libhello_1.0.0_filesystem")) # Check contents of one of the dependencies to make even surer -assert_contents(os.path.join("..", deps_dir, "hello_1.0.1_filesystem"), - ['../deps/hello_1.0.1_filesystem/alire', - '../deps/hello_1.0.1_filesystem/alire.toml', - '../deps/hello_1.0.1_filesystem/config', - '../deps/hello_1.0.1_filesystem/config/hello_config.ads', - '../deps/hello_1.0.1_filesystem/config/hello_config.gpr', - '../deps/hello_1.0.1_filesystem/config/hello_config.h', - '../deps/hello_1.0.1_filesystem/hello.gpr', - '../deps/hello_1.0.1_filesystem/src', - '../deps/hello_1.0.1_filesystem/src/hello.adb']) +assert_contents(base := os.path.join(vault_dir, "hello_1.0.1_filesystem"), + [f'{base}/alire', + f'{base}/alire.toml', + f'{base}/alire/complete_copy', + f'{base}/hello.gpr', + f'{base}/src', + f'{base}/src/hello.adb']) + +# Check the contents in the build dir, that should include generated configs + +# We need to find the hash first +base = glob.glob(os.path.join(build_dir, "hello_1.0.1_filesystem_*"))[0] + +assert_contents(base, + [f'{base}/alire', + f'{base}/alire.toml', + f'{base}/alire/complete_copy', + f'{base}/config', + f'{base}/config/hello_config.ads', + f'{base}/config/hello_config.gpr', + f'{base}/config/hello_config.h', + f'{base}/hello.gpr', + f'{base}/src', + f'{base}/src/hello.adb']) # And that the crate usual cache dir doesn't exist assert not os.path.exists(alr_workspace_cache()) diff --git a/testsuite/tests/dockerized/misc/default-cache/test.py b/testsuite/tests/dockerized/misc/default-cache/test.py index a1fc4b70..068063ef 100644 --- a/testsuite/tests/dockerized/misc/default-cache/test.py +++ b/testsuite/tests/dockerized/misc/default-cache/test.py @@ -4,8 +4,10 @@ it should. """ import os +import sys -from drivers.alr import alr_with, init_local_crate +from drivers.alr import alr_with, init_local_crate, run_alr +from drivers.helpers import contents # Forcing the deployment of a binary crate triggers the use of the global # cache, which should be created at the expected location. @@ -14,9 +16,29 @@ alr_with("gnat_native") home = os.environ["HOME"] +base = f"{home}/.cache/alire" + assert \ - os.path.isdir(f"{home}/.cache/alire/dependencies/gnat_native_8888.0.0_99fa3a55"), \ - "Default cache not found at the expected location" + os.path.isdir(f"{base}/toolchains/gnat_native_8888.0.0_99fa3a55"), \ + f"Toolchain dir not found at the expected location: {contents(base)}" + +# Let's also check the rest of dirs for shared builds + +# First, prevent an attempt at downloading a real compiler +run_alr("toolchain", "--disable-assistant") + +run_alr("config", "--global", "--set", "dependencies.shared", "true") +alr_with("crate_real") # This release will go in the cache +# Read-only vault +assert \ + os.path.isdir(f"{base}/releases/crate_real_1.0.0_filesystem"), \ + f"Vault not found at the expected location: f{contents(base)}" + +# Shared builds +assert \ + os.path.isdir(f"{base}/builds/crate_real_1.0.0_filesystem_deadbeef"), \ + "Vault not found at the expected location: f{contents(base)" + # TODO: above hash will need updating once hash computation is in place print('SUCCESS') diff --git a/testsuite/tests/get/external-tool-dependency/test.py b/testsuite/tests/get/external-tool-dependency/test.py index a9b8ccb9..32ef8028 100644 --- a/testsuite/tests/get/external-tool-dependency/test.py +++ b/testsuite/tests/get/external-tool-dependency/test.py @@ -34,6 +34,8 @@ compare(dir_content, 'main_1.0.0_filesystem/alire/cache/dependencies', make_dep_dir, make_dep_dir + "/alire", + make_dep_dir + "/alire/complete_copy", + 'main_1.0.0_filesystem/alire/complete_copy', 'main_1.0.0_filesystem/alire/config.toml', 'main_1.0.0_filesystem/config', 'main_1.0.0_filesystem/config/main_config.ads', diff --git a/testsuite/tests/get/git-local/test.py b/testsuite/tests/get/git-local/test.py index de2b7410..24d1ff8e 100644 --- a/testsuite/tests/get/git-local/test.py +++ b/testsuite/tests/get/git-local/test.py @@ -19,6 +19,7 @@ compare(list(filter 'libfoo_1.0.0_9ddda32b/alire', 'libfoo_1.0.0_9ddda32b/alire.toml', 'libfoo_1.0.0_9ddda32b/alire/alire.lock', + 'libfoo_1.0.0_9ddda32b/alire/complete_copy', 'libfoo_1.0.0_9ddda32b/alire/config.toml', 'libfoo_1.0.0_9ddda32b/b', 'libfoo_1.0.0_9ddda32b/b/x', diff --git a/testsuite/tests/get/unpack-in-place/test.py b/testsuite/tests/get/unpack-in-place/test.py index b00bde74..7ad1831f 100644 --- a/testsuite/tests/get/unpack-in-place/test.py +++ b/testsuite/tests/get/unpack-in-place/test.py @@ -12,6 +12,7 @@ compare(contents('libhello_1.0.0_filesystem'), ['libhello_1.0.0_filesystem/alire', 'libhello_1.0.0_filesystem/alire.toml', 'libhello_1.0.0_filesystem/alire/alire.lock', + 'libhello_1.0.0_filesystem/alire/complete_copy', 'libhello_1.0.0_filesystem/alire/config.toml', 'libhello_1.0.0_filesystem/config', 'libhello_1.0.0_filesystem/config/libhello_config.ads', diff --git a/testsuite/tests/printenv/compiler-indirect/test.py b/testsuite/tests/printenv/compiler-indirect/test.py index 24a60e5c..31c0f004 100644 --- a/testsuite/tests/printenv/compiler-indirect/test.py +++ b/testsuite/tests/printenv/compiler-indirect/test.py @@ -35,7 +35,7 @@ s = re.escape(dir_separator()) # Ensure the compiler-set flag is in there: assert_match(".*export TEST_PATH=.*" f"printenv__compiler-indirect{s}alr-config{s}cache{s}" - f"dependencies{s}gnat_native_8888\.0\.0_99fa3a55{s}bin", + f"toolchains{s}gnat_native_8888\.0\.0_99fa3a55{s}bin", p.out) print('SUCCESS') diff --git a/testsuite/tests/toolchain/directories/test.py b/testsuite/tests/toolchain/directories/test.py index 6a69b55f..34c98be2 100644 --- a/testsuite/tests/toolchain/directories/test.py +++ b/testsuite/tests/toolchain/directories/test.py @@ -22,7 +22,7 @@ unk_re = "[0-9]+\.[0-9]+\.[0-9]+_[0-9a-f]{8}" # Unknown version + Unknown hash def config_path_re(crate): - return re.escape(f"{config_dir}/cache/dependencies/{crate}_") + unk_re + return re.escape(f"{config_dir}/cache/toolchains/{crate}_") + unk_re def check_content(crate): paths = contents(cache_dir, crate) diff --git a/testsuite/tests/workflows/actions-as-root/test.py b/testsuite/tests/workflows/actions-as-root/test.py index 094e7b07..61f7ac5d 100644 --- a/testsuite/tests/workflows/actions-as-root/test.py +++ b/testsuite/tests/workflows/actions-as-root/test.py @@ -38,6 +38,7 @@ Path('src/empty.adb').touch() p = run_alr('build', complain_on_error=False) assert_match(".*compilation of empty.adb failed.*", p.out) +# Post fetch shouldn't be here because fetch already happened and was deleted # Post build shouldn't be here because of build failure check_not_expected('./test_post_fetch') check_expected('./test_pre_build') -- 2.39.5