From 79a83539bb1cded99a03f3bb89d0f6ad26b6869a Mon Sep 17 00:00:00 2001 From: Seb M'Caw Date: Thu, 9 Jan 2025 11:23:04 +0000 Subject: [PATCH] feat: setting to override source archive download command (#1815) * Support command mocking on Windows * Add tests * Make archive download command configurable * Fix validation of distribution.override * Prevent configuring command settings locally * Remove redundant Download function * Add documentation * Self-review * Fix test on Windows when gunzip is only available from msys2 * Restructure documentation * Add erroneously omitted user-changes.md entry for #1745 * Update user-changes.md * Fix deprecated 'fail_on_error' in spellcheck workflow * Switch to US English spellings * Review --------- Co-authored-by: Alejandro R. Mosteo --- .github/workflows/spellcheck.yml | 2 +- doc/getting-started.md | 7 + doc/private-crates.md | 159 +++++++++++++++++ doc/publishing.md | 50 ++---- doc/user-changes.md | 37 +++- src/alire/alire-environment-loading.adb | 4 +- ...nt-formatting.adb => alire-formatting.adb} | 30 +++- ...nt-formatting.ads => alire-formatting.ads} | 24 ++- ...alire-origins-deployers-source_archive.adb | 58 +++++- src/alire/alire-os_lib-download.adb | 8 - src/alire/alire-os_lib-download.ads | 9 + src/alire/alire-os_lib-subprocess.adb | 2 +- src/alire/alire-settings-builtins.ads | 22 ++- src/alire/alire-settings-edit.adb | 41 ++++- src/alire/alire-settings-edit.ads | 11 +- src/alire/alire-settings.adb | 35 ++-- src/alire/alire-settings.ads | 35 ++-- src/alire/alire-spawn.adb | 37 +++- src/alire/alire-spawn.ads | 15 ++ .../alire-platforms-current__windows.adb | 46 +---- src/alr/alr-commands-edit.adb | 99 ++++------- src/alr/alr-commands-settings.adb | 4 +- testsuite/drivers/alr.py | 30 +++- testsuite/drivers/helpers.py | 165 ++++++++++++------ .../my_index/index/index.toml | 1 + .../index/li/libhello/libhello-1.0.0.toml | 10 ++ .../tests/get/custom-download-command/test.py | 93 ++++++++++ .../get/custom-download-command/test.yaml | 4 + .../tests/pin/branch-remote-protocols/test.py | 10 +- .../tests/publish/private-indexes/test.py | 4 +- testsuite/tests/settings/basics/test.py | 20 +++ .../air-gapping/my_index/crates/hello.tar | Bin 0 -> 10240 bytes .../air-gapping/my_index/crates/hello.tgz | Bin 334 -> 0 bytes .../air-gapping/my_index/crates/libhello.tar | Bin 0 -> 10240 bytes .../air-gapping/my_index/crates/libhello.tgz | Bin 428 -> 0 bytes .../my_index/index/he/hello/hello-1.0.1.toml | 2 +- .../index/li/libhello/libhello-1.0.0.toml | 2 +- testsuite/tests/workflows/air-gapping/test.py | 68 +++++--- .../tests/workflows/air-gapping/test.yaml | 3 - testsuite/tests/workflows/edit/test.py | 44 +++-- 40 files changed, 863 insertions(+), 328 deletions(-) create mode 100644 doc/private-crates.md rename src/alire/{alire-environment-formatting.adb => alire-formatting.adb} (85%) rename src/alire/{alire-environment-formatting.ads => alire-formatting.ads} (64%) create mode 100644 testsuite/tests/get/custom-download-command/my_index/index/index.toml create mode 100644 testsuite/tests/get/custom-download-command/my_index/index/li/libhello/libhello-1.0.0.toml create mode 100644 testsuite/tests/get/custom-download-command/test.py create mode 100644 testsuite/tests/get/custom-download-command/test.yaml create mode 100644 testsuite/tests/workflows/air-gapping/my_index/crates/hello.tar delete mode 100644 testsuite/tests/workflows/air-gapping/my_index/crates/hello.tgz create mode 100644 testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tar delete mode 100644 testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tgz diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 37bf251f..67f7a100 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -16,4 +16,4 @@ jobs: reporter: github-pr-annotations locale: "US" pattern: '*.md' - fail_on_error: true + fail_level: warning diff --git a/doc/getting-started.md b/doc/getting-started.md index 40b2441c..221e3a4d 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -273,6 +273,13 @@ website: * [alire.ada.dev](https://alire.ada.dev) +### Using Alire with other indexes + +So far in this guide we have been using the community index, a central catalog +of publicly available crates, but it is possible to host your own index for +crates which you do not wish to make generally available. For more information, +see [using Alire with private crates](private-crates). + ## Build environment To create a build environment, `alr` sets environment variables such as diff --git a/doc/private-crates.md b/doc/private-crates.md new file mode 100644 index 00000000..cdd46bda --- /dev/null +++ b/doc/private-crates.md @@ -0,0 +1,159 @@ +# Using Alire with private crates + +By default, Alire uses the open-source crates available through the community +index, but it can also be configured to fetch non-public crates (e.g. during +development or when working with proprietary code). There are two key components +required to make a crate available to Alire; the crate must be searchable in an +*index* (though this requirement can be temporarily circumvented with +[pins](catalog-format-spec#work-in-progress-dependency-overrides)), and the +crate's source files must be fetchable from an *origin*. + + +## Using a private index + +### Creating an index + +An Alire index is nothing more than a directory containing a collection of +release manifest files in a certain directory structure. Full details of the +format of an index can be found in +[the catalog format specification](catalog-format-spec). + +To create a new index, simply create an empty directory at a location of your +choice, and add to it a file called `index.toml` containing one line with the +form `version = "x.x.x"`, specifying the index format used. The range of +versions Alire is compatible with can be found by running `alr version`; when +creating a new index you should simply use the highest version listed under +`compatible index versions:`. + +If future updates to Alire affect compatibility with existing indexes, they will +be listed in +[BREAKING.md](https://github.com/alire-project/alire/blob/master/BREAKING.md). + +### Configuring an index + +To start using your new index, run +```sh +alr index --add= --name= +``` +where `` is a human-friendly label that `alr` will use to refer to it. + +It is possible to configure multiple indexes (with any conflicts resolved using +a priority order specified by the `--before` switch). For example, you may wish +to configure both the community index and a private index of your own +unpublished crates. + +If you intend to use your private index as the only index (i.e. without also +keeping the community index configured), you will need to add crates for the +compiler and GPRbuild. See the community index's +[`gnat_*` crates](https://github.com/alire-project/alire-index/tree/HEAD/index/gn) and +[`gprbuild` crate](https://github.com/alire-project/alire-index/tree/HEAD/index/gp/gprbuild) +for more details on how this can be achieved. + +### Adding a new crate to an index + +In order to add a crate to the index, you must create a suitable manifest file +to describe the release, and place this manifest at the appropriate location in +the index directory. + +The creation of the manifest file is automated through the `alr publish` +command; submission to the community index can be disabled by supplying the +`--for-private-index` switch. See [publishing](publishing) for more detail on +this process. + +The resulting manifest file must then be copied to the index directory. The +`alr publish` command will provide instructions on the correct location at which +to place it. + +The newly added crate will become available for use with `alr` immediately, +unless the crate being published contains `"provides"` definitions, in which +case a call to `alr index --update-all` will be required. + +### Remote indexes + +You may wish to share your private index outside of your local filesystem. Any +means of synchronizing the contents of the index directory will suffice, but +Alire will manage this process automatically if you use a remote Git repository. + +It is often useful to have other files in the same repository as the index +(a `README`, CI configuration, templates etc.), so the index itself is located +in a first-level subdirectory of the repository (conventionally called `index/`, +though `alr` searches for any directory containing an `index.toml` file). + +To start using a remote Git repository as an index, run +```sh +alr index --add= --name= +``` +Note that `` can point directly to the remote repository, so no local clone is required. + +Changes on the remote index will not take effect until `alr` performs an index +update, either with `alr index --update-all`, or through a scheduled auto-update +(which is performed every 24 hours by default). + +Once configured, there is no practical difference between the community index +and a private index on your own remote Git repository. + + +## Requiring user authentication + +In addition to listing crates in a private index, you may wish to limit access +to authenticated users only. This is supported for crates and indexes made +available as Git repositories, and for crates made available as archive files. + +### Git repositories + +Crates and indexes can be fetched as clones of a git repository, and for most +users this option will require minimal additional configuration. + +Repositories are cloned and updated by calling out to the system's `git` +command, so `git` must be configured with appropriate credentials to +authenticate with any private origin. This usually means either Git-over-SSH +(see your SSH client's documentation), or HTTPS with `git`'s credential handling +(documented [here](https://git-scm.com/docs/gitcredentials)). As a general rule, +if a `git clone` or `git fetch` on a crate/index origin succeeds, then so will +any corresponding dependency updates with `alr`. + +The main consideration when using such repositories with Alire is ensuring that +the correct form of URL is used. It must be apparent to `alr` that the URL +refers to a Git repository (which can be disambiguated with a `git+` prefix), +and `git` must be able to infer which protocol should be used (documented +[here](https://git-scm.com/docs/git-clone#URLS)). It is therefore recommended +that URLs use the schemes `git+https://` or `git+ssh://` as appropriate. + +Repositories on the local filesystem can be specified with the form +`git+file:/some/path`, though this is not recommended except as a temporary +arrangement for testing purposes. + +### Source archives + +Crates can also be fetched as an archive file (either tarball or Zip) containing +the relevant sources. `alr` simply requires some means of downloading this file +from a URL to a local filesystem location. + +The command used to fetch such files is specified by the [settings](settings) +key `origins.archive.download_cmd`. By default `alr` uses the command +`curl ${URL} -L -s -o ${DEST}`, which does not attempt any form of +authentication, but this can be changed to any equivalent alternative which +implements a desired authentication scheme. The command should download the file +to `${DEST}` (the full file path, not a directory), and must not pollute the +containing directory with other files. + +The simplest way to enable user authentication is to configure the server to +accept HTTP Basic authentication and supply `curl` with the switch `--netrc` +(or equivalently `-n`), which instructs it to scan the `.netrc` file in the +user's home directory and attempt to authenticate with any credentials found +there (see `man curl` and `man netrc` for more detail). This is achieved by +invoking +```sh +alr settings --set --global origins.archive.download_cmd 'curl ${URL} -n -L -s -o ${DEST}' +``` +Note that most terminals will perform substitutions on double-quoted strings +containing `${SOMETHING}`, so it is important to enclose the value in +single-quotes. + +If you wish to use a `.wgetrc` configuration file instead, the equivalent `wget` +command is `'wget ${URL} -q -O ${DEST}'`. + +This setting only accepts a simple space-separated command, with no scripting +functionality. If this is not sufficient, you can write more complex logic (or +commands with arguments containing spaces) to a separate script, for instance +by setting the value to `'python /path/to/my_script.py ${URL} ${DEST}'`. diff --git a/doc/publishing.md b/doc/publishing.md index 23ab40ca..fbaaeae1 100644 --- a/doc/publishing.md +++ b/doc/publishing.md @@ -314,36 +314,22 @@ This will be shown as: ## Publishing to a local/private index Having a local or private index may be useful sometimes, be it for local -testing, or for private crates not intended for publication. +testing, or for private crates not intended for publication. For more +information on private indexes, see +[this guide](private-crates#using-a-private-index). -There is no practical difference between the community index and a private index -stored locally on disk or on your own infrastructure. An index must be located -in a first level subdirectory of an accessible git repository or local -filesystem location (or optionally at the top level in the case of a local -filesystem index). This subdirectory should contain only an `index.toml` -file and one or more `cr/crate_name` subdirectories within which the crate -manifests themselves are located. The `index.toml` file contains one line with -the form `version = "x.x.x"`, specifying the index format used. The range of -versions Alire is compatible with can be found by running `alr version`, and -breaking changes are listed in -[BREAKING.md](https://github.com/alire-project/alire/blob/master/BREAKING.md). - -To start using such an index, run - -`alr index --add= --name=`, - -where `` is a human-friendly label that `alr` will use to refer to it. - -To publish a crate to a private index, run - -`alr publish --for-private-index [ ]` - -as described in the sections above, then place the manifest file it generates at -the indicated path (relative to the location of `index.toml`). - -Additions to indexes stored locally on the disk will take effect immediately, -unless the crate being published contains `"provides"` definitions, in which -case an index update will be required (either with `alr index --update-all`, or -through a scheduled auto-update) to ensure it is properly used by the dependency -solver. An index update will always be required when publishing to a git -repository index. +To "publish" a crate to a private index, run +``` +alr publish --for-private-index [ []] +``` +where the `--for-private-index` switch disables the submission step and certain +checks which are only applicable to the community index, and the remaining +arguments function as described above. This will generate a manifest file which +you can place at the indicated path (relative to the location of `index.toml`) +in your private index. + +One important thing to note is that publishing from a local repository will +detect the URL configured as the Git remote (as displayed by +`git remote show origin`). If this is not configured with the recommended form +(as discussed [here](private-crates#git-repositories)), you may wish to pass the +desired URL explicitly. diff --git a/doc/user-changes.md b/doc/user-changes.md index 50f6438e..8fbb0c25 100644 --- a/doc/user-changes.md +++ b/doc/user-changes.md @@ -6,9 +6,28 @@ stay on top of `alr` new features. ## Release `2.1` +### Custom download command for archive crates + +PR [#1815](https://github.com/alire-project/alire/pull/1815) + +The command used to download a crate as a source archive can now be configured +using the `origins.archive.download_cmd` key of `alr settings`, instead of using +a hard-coded `curl` command. The tokens `${URL}` and `${DEST}` are replaced by +the origin URL and destination file path respectively. + +For example +```sh +alr settings --set --global origins.archive.download_cmd 'curl ${URL} --netrc -L -s -o ${DEST}' +``` +configures `alr` to use the default command with the addition of the switch +`--netrc` (which instructs `curl` to use the login information in the `.netrc` +file found in the user's home directory). + +The default behavior is unchanged. + ### Abbreviated `--tree` output for repeating dependencies -PR [1814](https://github.com/alire-project/alire/pull/1814) +PR [#1814](https://github.com/alire-project/alire/pull/1814) By default, repeated dependencies are now omitted by `--tree` output, e.g.: @@ -40,7 +59,7 @@ switch. ### Faster `alr search` without resolving dependencies -PR [1799](https://github.com/alire-project/alire/pull/1799) +PR [#1799](https://github.com/alire-project/alire/pull/1799) `alr search` no longer solves dependencies of releases by default, in order to speed up the command. The `--solve` switch can be used to achieve the old @@ -51,11 +70,21 @@ In the new default situation, releases that have dependencies are marked with a dependencies and replace the '?' with either nothing for a solvable release or the usual 'X' if dependencies are unsatisfiable. +### Support for private indexes with `alr publish --for-private-index` + +PR [#1745](https://github.com/alire-project/alire/pull/1745) + +Automated manifest generation with `alr publish` can now be performed for crates +which are not intended for submission to the community index by supplying the +`--for-private-index` switch. This has the same effects as `--skip-submit`, and +additionally disables a number of checks that enforce submission requirements +specific to the community index. + ## Release `2.0` ### `ALIRE_SETTINGS_DIR` replaces `ALR_CONFIG` -PR [1625](https://github.com/alire-project/alire/pull/1625) +PR [#1625](https://github.com/alire-project/alire/pull/1625) This reflects the new nomenclature of Alire settings versus crate configuration. Also, it better reflects that the effect is on the whole library @@ -63,7 +92,7 @@ and not only the `alr` command-line tool. ### `alr settings` replaces `alr config` -PR [1617](https://github.com/alire-project/alire/pull/1617) +PR [#1617](https://github.com/alire-project/alire/pull/1617) The `alr settings` command replaces the `alr config` command. This change is introduced to tackle the confusion between the configuration of the Alire diff --git a/src/alire/alire-environment-loading.adb b/src/alire/alire-environment-loading.adb index 16890a0e..18ac8264 100644 --- a/src/alire/alire-environment-loading.adb +++ b/src/alire/alire-environment-loading.adb @@ -1,5 +1,5 @@ with Alire_Early_Elaboration; -with Alire.Environment.Formatting; +with Alire.Formatting; with Alire.GPR; with Alire.Platforms.Current; with Alire.Properties.Scenarios; @@ -125,7 +125,7 @@ package body Alire.Environment.Loading is Formatting.Format (Act.Value, Formatting.For_Manifest_Environment (Release_Base), - Is_Path => True); + Convert_Path_Seps => True); begin case Act.Action is diff --git a/src/alire/alire-environment-formatting.adb b/src/alire/alire-formatting.adb similarity index 85% rename from src/alire/alire-environment-formatting.adb rename to src/alire/alire-formatting.adb index 5d7105c4..35f8046d 100644 --- a/src/alire/alire-environment-formatting.adb +++ b/src/alire/alire-formatting.adb @@ -5,7 +5,7 @@ with Ada.Strings.Unbounded; use Ada.Strings.Unbounded; with Alire.OS_Lib; with Alire.Platforms.Current; -package body Alire.Environment.Formatting is +package body Alire.Formatting is -------------- -- Contains -- @@ -22,6 +22,22 @@ package body Alire.Environment.Formatting is function Value (This : Replacements; Pattern : Patterns) return String is (This (Pattern)); + -------------------------- + -- For_Archive_Download -- + -------------------------- + + function For_Archive_Download (URL : String; + Destination : Absolute_Path) + return Replacements + is + Result : Replacements; + begin + Result.Insert (Formatting.URL, URL); + Result.Insert (Formatting.Dest, Destination); + + return Result; + end For_Archive_Download; + ------------------------------ -- For_Manifest_Environment -- ------------------------------ @@ -98,9 +114,9 @@ package body Alire.Environment.Formatting is -- Format -- ------------ - function Format (Item : String; - Repl : Replacements; - Is_Path : Boolean) + function Format (Item : String; + Repl : Replacements; + Convert_Path_Seps : Boolean) return String is ------------- @@ -121,7 +137,7 @@ package body Alire.Environment.Formatting is Replace_Slice (Str, From, To, "TEST"); else - raise Unknown_Formatting_Key; + raise Unknown_Formatting_Key with "Unknown formatting key: " & Id; end if; end Replace; @@ -170,11 +186,11 @@ package body Alire.Environment.Formatting is -- For final usage, we use the native separator - if Is_Path then + if Convert_Path_Seps then return To_Native (+Result); else return +Result; end if; end Format; -end Alire.Environment.Formatting; +end Alire.Formatting; diff --git a/src/alire/alire-environment-formatting.ads b/src/alire/alire-formatting.ads similarity index 64% rename from src/alire/alire-environment-formatting.ads rename to src/alire/alire-formatting.ads index 24d3d4b9..a22fc585 100644 --- a/src/alire/alire-environment-formatting.ads +++ b/src/alire/alire-formatting.ads @@ -2,11 +2,13 @@ with Alire.Roots; private with Ada.Containers.Indefinite_Ordered_Maps; -package Alire.Environment.Formatting is +package Alire.Formatting is type Patterns is (Crate_Root, + Dest, Distrib_Root, - GPR_File); + GPR_File, + URL); -- These correspond directly with ${PATTERNs} that can be replaced function Dollar_Image (Pattern : Patterns) return String @@ -14,6 +16,10 @@ package Alire.Environment.Formatting is type Replacements (<>) is tagged private; + function For_Archive_Download (URL : String; + Destination : Absolute_Path) + return Replacements; + function For_Manifest_Environment (Crate_Root : Any_Path) return Replacements; -- Crate_Root can't be an absolute path as this may be called with relative @@ -28,12 +34,14 @@ package Alire.Environment.Formatting is function Value (This : Replacements; Pattern : Patterns) return String with Pre => This.Contains (Pattern); - function Format (Item : String; - Repl : Replacements; - Is_Path : Boolean) + function Format (Item : String; + Repl : Replacements; + Convert_Path_Seps : Boolean) return String; - -- If Is_Path, a final pass is done to use platform-specific dir separators - -- Format the item with ${} replacement patterns. + -- Format Item with ${} replacement patterns. + -- + -- If Convert_Path_Seps, a final pass is done to replace forward slashes + -- with native slashes on Windows, unless they are an escape sequence. Unknown_Formatting_Key : exception; @@ -44,4 +52,4 @@ private type Replacements is new Pattern_String_Maps.Map with null record; -end Alire.Environment.Formatting; +end Alire.Formatting; diff --git a/src/alire/alire-origins-deployers-source_archive.adb b/src/alire/alire-origins-deployers-source_archive.adb index fa06f24f..137fb35b 100644 --- a/src/alire/alire-origins-deployers-source_archive.adb +++ b/src/alire/alire-origins-deployers-source_archive.adb @@ -4,8 +4,10 @@ with AAA.Strings; use AAA.Strings; with Alire.Errors; with Alire.Directories; +with Alire.Formatting; with Alire.OS_Lib.Subprocess; -with Alire.OS_Lib.Download; +with Alire.Settings.Builtins; +with Alire.Spawn; with Alire.VFS; with Alire.URI; with Alire.Utils; use Alire.Utils; @@ -126,6 +128,54 @@ package body Alire.Origins.Deployers.Source_Archive is return Hashes.Digest (Hashes.Hash_File (Kind, Archive_File)); end Compute_Hash; + -------------- + -- Download -- + -------------- + -- Download the source archive from a non-local origin. + + function Download (URL : String; + Filename : Any_Path; + Folder : Directory_Path) + return Outcome + is + use GNATCOLL.VFS; + use Alire.Directories; + Archive_File : constant File_Path := + Folder / Ada.Directories.Simple_Name (Filename); + + Configured_Download_Cmd : constant String := + Alire.Settings.Builtins.Origins_Archive_Download_Cmd.Get; + + -- In the case of the default download command, we adjust curl's + -- verbosity if warranted by the logging level. + Download_Cmd : constant String := + (if Log_Level >= Trace.Info + and then Configured_Download_Cmd = "curl ${URL} -L -s -o ${DEST}" + then "curl ${URL} -L --progress-bar -o ${DEST}" + else Configured_Download_Cmd); + + procedure Exec_Check (Exec : String) is + begin + if Exec = "curl" then + Utils.Tools.Check_Tool (Utils.Tools.Curl); + end if; + end Exec_Check; + begin + Trace.Debug ("Creating folder: " & Folder); + Create (+Folder).Make_Dir; + + Trace.Detail ("Downloading file: " & URL); + Alire.Spawn.Settings_Command + (Download_Cmd, + Formatting.For_Archive_Download (URL, Archive_File), + Exec_Check'Access); + + return Outcome_Success; + exception + when E : others => + return Alire.Errors.Get (E); + end Download; + ----------- -- Fetch -- ----------- @@ -149,9 +199,9 @@ package body Alire.Origins.Deployers.Source_Archive is return Outcome_Success; else - return OS_Lib.Download.File (URL => This.Base.Archive_URL, - Filename => This.Base.Archive_Name, - Folder => Folder); + return Download (URL => This.Base.Archive_URL, + Filename => This.Base.Archive_Name, + Folder => Folder); end if; end Fetch; diff --git a/src/alire/alire-os_lib-download.adb b/src/alire/alire-os_lib-download.adb index fe814438..c287b61e 100644 --- a/src/alire/alire-os_lib-download.adb +++ b/src/alire/alire-os_lib-download.adb @@ -1,11 +1,7 @@ with Ada.Directories; -with AAA.Strings; use AAA.Strings; - with Alire.Errors; with Alire.OS_Lib.Subprocess; -with Alire.Utils; use Alire.Utils; -with Alire.Utils.Tools; with GNATCOLL.VFS; @@ -25,10 +21,6 @@ package body Alire.OS_Lib.Download is Archive_File : constant Directory_Path := Folder / Ada.Directories.Simple_Name (Filename); begin - - -- Make sure curl is installed - Utils.Tools.Check_Tool (Utils.Tools.Curl); - Trace.Debug ("Creating folder: " & Folder); Create (+Folder).Make_Dir; diff --git a/src/alire/alire-os_lib-download.ads b/src/alire/alire-os_lib-download.ads index 46b4b94f..b58649ec 100644 --- a/src/alire/alire-os_lib-download.ads +++ b/src/alire/alire-os_lib-download.ads @@ -5,5 +5,14 @@ package Alire.OS_Lib.Download is Filename : Any_Path; Folder : Directory_Path) return Outcome; + -- Download a single file using `curl` + -- + -- Specifically, uses 'curl --location --silent --output ' + -- (except that --silent may be replaced by --progress-bar depending on the + -- log level). + -- + -- Note that this doesn't check that 'curl' is available (calling code may + -- wish to use 'Utils.Tools.Check_Tool'), nor that the destination file + -- doesn't already exist. end Alire.OS_Lib.Download; diff --git a/src/alire/alire-os_lib-subprocess.adb b/src/alire/alire-os_lib-subprocess.adb index 7ef0dbd8..5b855340 100644 --- a/src/alire/alire-os_lib-subprocess.adb +++ b/src/alire/alire-os_lib-subprocess.adb @@ -141,7 +141,7 @@ package body Alire.OS_Lib.Subprocess is Raise_Checked_Error ("Command " & Image (Command, Arguments) - & " exited with code" & AAA.Strings.Trim (Exit_Code'Image) + & " exited with code " & AAA.Strings.Trim (Exit_Code'Image) & " and output: " & Output.Flatten (Separator => "\n")); return Output; diff --git a/src/alire/alire-settings-builtins.ads b/src/alire/alire-settings-builtins.ads index 70b144c8..1444f082 100644 --- a/src/alire/alire-settings-builtins.ads +++ b/src/alire/alire-settings-builtins.ads @@ -56,10 +56,11 @@ package Alire.Settings.Builtins is -- EDITOR Editor_Cmd : constant Builtin := New_Builtin - (Key => "editor.cmd", - Kind => Stn_String, - Def => "", - Help => + (Key => "editor.cmd", + Kind => Stn_String, + Def => "", + Global_Only => True, + Help => "Editor command and arguments for editing crate code (alr edit)." & " The executables and arguments are separated by a single space" & " character. The token ${GPR_FILE} is replaced by" & @@ -112,6 +113,19 @@ package Alire.Settings.Builtins is Def => "alire-index", Help => "Name of the index repository."); + -- ORIGINS + + Origins_Archive_Download_Cmd : constant Builtin := New_Builtin + (Key => "origins.archive.download_cmd", + Kind => Stn_String, + Def => "curl ${URL} -L -s -o ${DEST}", + Global_Only => True, + Help => + "The command used to download crates which are published as archives." + & " The executables and arguments are separated by a single space" + & " character. The token ${DEST} is replaced by the destination path," + & " and ${URL} by the URL to download."); + -- SOLVER Solver_Autonarrow : constant Builtin := New_Builtin diff --git a/src/alire/alire-settings-edit.adb b/src/alire/alire-settings-edit.adb index dcc75117..02bf7c11 100644 --- a/src/alire/alire-settings-edit.adb +++ b/src/alire/alire-settings-edit.adb @@ -193,7 +193,7 @@ package body Alire.Settings.Edit is CLIC.Config.Load.From_TOML (C => DB_Instance, Origin => Lvl'Img, Path => Filepath (Lvl), - Check => Valid_Builtin'Access); + Check => Valid_Builtin_Check (Lvl)); end if; end loop; @@ -266,8 +266,9 @@ package body Alire.Settings.Edit is -- Valid_Builtin -- ------------------- - function Valid_Builtin (Key : CLIC.Config.Config_Key; - Value : TOML_Value) + function Valid_Builtin (Key : CLIC.Config.Config_Key; + Value : TOML_Value; + Global : Boolean) return Boolean is Result : Boolean := True; @@ -311,7 +312,13 @@ package body Alire.Settings.Edit is and then Utils.Is_Valid_GitHub_Username (Value.As_String); end case; - exit when not Result; + -- Error if Global_Only being set locally + + if Result and then Ent.Global_Only and then not Global then + Trace.Error + ("Configuration key '" & Key & "' must be set globally."); + return False; + end if; -- Apply the own builtin check if any. @@ -321,6 +328,7 @@ package body Alire.Settings.Edit is ("Invalid value '" & CLIC.Config.Image (Value) & "' for builtin configuration '" & Key & "'. " & "Specific builtin check failed."); + return False; end if; end if; @@ -338,6 +346,31 @@ package body Alire.Settings.Edit is return Result; end Valid_Builtin; + -------------------------- + -- Valid_Global_Builtin -- + -------------------------- + + function Valid_Global_Builtin + (Key : CLIC.Config.Config_Key; Value : TOML_Value) return Boolean + is (Valid_Builtin (Key, Value, Global => True)); + + ------------------------- + -- Valid_Local_Builtin -- + ------------------------- + + function Valid_Local_Builtin + (Key : CLIC.Config.Config_Key; Value : TOML_Value) return Boolean + is (Valid_Builtin (Key, Value, Global => False)); + + ------------------------- + -- Valid_Builtin_Check -- + ------------------------- + + function Valid_Builtin_Check (Lvl : Level) return CLIC.Config.Check_Import + is (case Lvl is + when Global => Valid_Global_Builtin'Access, + when Local => Valid_Local_Builtin'Access); + ------------------- -- Builtins_Info -- ------------------- diff --git a/src/alire/alire-settings-edit.ads b/src/alire/alire-settings-edit.ads index 495e644e..7440fc17 100644 --- a/src/alire/alire-settings-edit.ads +++ b/src/alire/alire-settings-edit.ads @@ -70,10 +70,17 @@ package Alire.Settings.Edit is procedure Print_Builtins_Doc; -- Print a Markdown documentation for the built-in settings options - function Valid_Builtin (Key : CLIC.Config.Config_Key; - Value : TOML.TOML_Value) + function Valid_Builtin (Key : CLIC.Config.Config_Key; + Value : TOML.TOML_Value; + Global : Boolean) return Boolean; -- Check that the combination satisfies builtin rules + -- + -- Global_Only settings will be ignored (with an error message) if Global + -- is False. + + function Valid_Builtin_Check (Lvl : Level) return CLIC.Config.Check_Import; + -- Return the appropriate Check_Import procedure for a given Level private diff --git a/src/alire/alire-settings.adb b/src/alire/alire-settings.adb index 41bad57f..635c94aa 100644 --- a/src/alire/alire-settings.adb +++ b/src/alire/alire-settings.adb @@ -144,20 +144,22 @@ package body Alire.Settings is -- New_Builtin -- ----------------- - function New_Builtin (Key : CLIC.Config.Config_Key; - Kind : Builtin_Kind; - Def : String := ""; - Help : String := ""; - Public : Boolean := True; - Check : CLIC.Config.Check_Import := null) + function New_Builtin (Key : CLIC.Config.Config_Key; + Kind : Builtin_Kind; + Def : String := ""; + Help : String := ""; + Public : Boolean := True; + Global_Only : Boolean := False; + Check : CLIC.Config.Check_Import := null) return Builtin_Option is begin - return Result : constant Builtin_Option := (Key => +Key, - Kind => Kind, - Def => +Def, - Help => +Help, - Check => Check) + return Result : constant Builtin_Option := (Key => +Key, + Kind => Kind, + Def => +Def, + Help => +Help, + Global_Only => Global_Only, + Check => Check) do if Public then All_Builtins.Insert (Key, Result); @@ -169,11 +171,12 @@ package body Alire.Settings is -- New_Builtin -- ----------------- - function New_Builtin (Key : CLIC.Config.Config_Key; - Def : Boolean; - Help : String := ""; - Public : Boolean := True; - Check : CLIC.Config.Check_Import := null) + function New_Builtin (Key : CLIC.Config.Config_Key; + Def : Boolean; + Help : String := ""; + Public : Boolean := True; + Global_Only : Boolean := False; + Check : CLIC.Config.Check_Import := null) return Builtin_Option is (New_Builtin (Key => Key, Kind => Stn_Bool, diff --git a/src/alire/alire-settings.ads b/src/alire/alire-settings.ads index b04fa932..6ad48853 100644 --- a/src/alire/alire-settings.ads +++ b/src/alire/alire-settings.ads @@ -63,20 +63,22 @@ package Alire.Settings is procedure Unset (This : Builtin_Option; Level : Settings.Level); - function New_Builtin (Key : CLIC.Config.Config_Key; - Kind : Builtin_Kind; - Def : String := ""; - Help : String := ""; - Public : Boolean := True; - Check : CLIC.Config.Check_Import := null) + function New_Builtin (Key : CLIC.Config.Config_Key; + Kind : Builtin_Kind; + Def : String := ""; + Help : String := ""; + Public : Boolean := True; + Global_Only : Boolean := False; + Check : CLIC.Config.Check_Import := null) return Builtin_Option with Pre => Help /= "" or else not Public; - function New_Builtin (Key : CLIC.Config.Config_Key; - Def : Boolean; - Help : String := ""; - Public : Boolean := True; - Check : CLIC.Config.Check_Import := null) + function New_Builtin (Key : CLIC.Config.Config_Key; + Def : Boolean; + Help : String := ""; + Public : Boolean := True; + Global_Only : Boolean := False; + Check : CLIC.Config.Check_Import := null) return Builtin_Option with Pre => Help /= "" or else not Public; @@ -88,11 +90,12 @@ private -- The Alire user settings database type Builtin_Option is tagged record - Key : Ada.Strings.Unbounded.Unbounded_String; - Kind : Builtin_Kind; - Def : Ada.Strings.Unbounded.Unbounded_String; - Help : Ada.Strings.Unbounded.Unbounded_String; - Check : CLIC.Config.Check_Import := null; + Key : Ada.Strings.Unbounded.Unbounded_String; + Kind : Builtin_Kind; + Def : Ada.Strings.Unbounded.Unbounded_String; + Help : Ada.Strings.Unbounded.Unbounded_String; + Check : CLIC.Config.Check_Import := null; + Global_Only : Boolean := False; end record; -------------- diff --git a/src/alire/alire-spawn.adb b/src/alire/alire-spawn.adb index fb5369e2..e0560fbb 100644 --- a/src/alire/alire-spawn.adb +++ b/src/alire/alire-spawn.adb @@ -7,9 +7,10 @@ package body Alire.Spawn is -- Command -- ------------- - procedure Command (Cmd : String; - Args : AAA.Strings.Vector; - Understands_Verbose : Boolean := False) + procedure Command + (Cmd : String; + Args : AAA.Strings.Vector; + Understands_Verbose : Boolean := False) is Unused_Output : AAA.Strings.Vector; begin @@ -23,12 +24,38 @@ package body Alire.Spawn is end if; end Command; + ---------------------- + -- Settings_Command -- + ---------------------- + + procedure Settings_Command + (Cmd : String; + Replacements : Alire.Formatting.Replacements; + Exec_Check : access procedure (Exec : String) := null) + is + Args : AAA.Strings.Vector := AAA.Strings.Split (Cmd, ' '); + Exec : constant String := Args.First_Element; + Replaced_Args : AAA.Strings.Vector; + begin + if Exec_Check /= null then + Exec_Check (Exec); + end if; + if Alire.OS_Lib.Subprocess.Locate_In_Path (Exec) = "" then + Raise_Checked_Error ("'" & Exec & "' not available or not in PATH."); + end if; + Args.Delete_First; + for Element of Args loop + Replaced_Args.Append + (Alire.Formatting.Format (Element, Replacements, False)); + end loop; + Command (Exec, Replaced_Args); + end Settings_Command; + -------------- -- Gprbuild -- -------------- - procedure Gprbuild (Project_File : String; - Extra_Args : AAA.Strings.Vector) + procedure Gprbuild (Project_File : String; Extra_Args : AAA.Strings.Vector) is use AAA.Strings; begin diff --git a/src/alire/alire-spawn.ads b/src/alire/alire-spawn.ads index 994d39b2..a14fa245 100644 --- a/src/alire/alire-spawn.ads +++ b/src/alire/alire-spawn.ads @@ -1,5 +1,6 @@ with AAA.Strings; +with Alire.Formatting; with Alire.Releases; package Alire.Spawn is @@ -14,6 +15,20 @@ package Alire.Spawn is -- Adds -v if understands in Debug log level -- Summary is shown after process successful end, if Log_Level = Info + procedure Settings_Command + (Cmd : String; + Replacements : Alire.Formatting.Replacements; + Exec_Check : access procedure (Exec : String) := null); + -- Launches a command from a string, according to the conventions used by + -- commands configurable with `alr settings`. + -- + -- Parses a space-separated string and performs ${} pattern replacements + -- (on arguments only). + -- + -- Exec_Check is called with the executable name between parsing Cmd into + -- arguments and executing the command, e.g. to check that the executable + -- is available. + procedure Gprbuild (Project_File : String; Extra_Args : AAA.Strings.Vector); diff --git a/src/alire/os_windows/alire-platforms-current__windows.adb b/src/alire/os_windows/alire-platforms-current__windows.adb index 8ec2f3fd..3dc98efa 100644 --- a/src/alire/os_windows/alire-platforms-current__windows.adb +++ b/src/alire/os_windows/alire-platforms-current__windows.adb @@ -8,11 +8,10 @@ pragma Unreferenced (Alire_Early_Elaboration); with Alire.Environment; with Alire.OS_Lib; use Alire.OS_Lib; +with Alire.OS_Lib.Download; with Alire.Settings.Builtins.Windows; with Alire.Errors; -with GNATCOLL.VFS; - with CLIC.User_Input; package body Alire.Platforms.Current is @@ -220,43 +219,6 @@ package body Alire.Platforms.Current is is use AAA.Strings; - ------------------- - -- Download_File -- - ------------------- - - function Download_File (URL : String; - Filename : Any_Path; - Folder : Directory_Path) - return Outcome - is - use GNATCOLL.VFS; - - Archive_File : constant Directory_Path := - Folder / Ada.Directories.Simple_Name (Filename); - begin - - Trace.Debug ("Creating folder: " & Folder); - Create (+Folder).Make_Dir; - - Trace.Detail ("Downloading file: " & URL); - - OS_Lib.Subprocess.Checked_Spawn - ("curl", - Empty_Vector & - URL & - "--location" & -- allow for redirects at the remote host - (if Log_Level < Trace.Info - then Empty_Vector & "--silent" - else Empty_Vector & "--progress-bar") & - "--output" & - Archive_File); - - return Outcome_Success; - exception - when E : others => - return Alire.Errors.Get (E); - end Download_File; - Msys2_Installer : constant String := Settings.Builtins.Windows.Msys2_Installer.Get; @@ -276,9 +238,9 @@ package body Alire.Platforms.Current is with "Attempting to install msys2 during testsuite run"; end if; - Result := Download_File (Msys2_Installer_URL, - Msys2_Installer, - Install_Dir); + Result := Alire.OS_Lib.Download.File (Msys2_Installer_URL, + Msys2_Installer, + Install_Dir); if not Result.Success then return Result; end if; diff --git a/src/alr/alr-commands-edit.adb b/src/alr/alr-commands-edit.adb index 95bf4182..91abb03b 100644 --- a/src/alr/alr-commands-edit.adb +++ b/src/alr/alr-commands-edit.adb @@ -1,8 +1,9 @@ with Ada.Containers; with Alire; use Alire; -with Alire.Environment.Formatting; +with Alire.Formatting; with Alire.Settings.Builtins; +with Alire.Spawn; with Alire.OS_Lib.Subprocess; with Alire.Platforms.Current; @@ -10,7 +11,7 @@ with CLIC.User_Input; package body Alr.Commands.Edit is - package Format renames Alire.Environment.Formatting; + package Format renames Alire.Formatting; Switch_Select : constant String := "--select-editor"; @@ -34,28 +35,31 @@ package body Alr.Commands.Edit is Put_Info ("`alr edit " & Switch_Select & "`"); end Set_Config_Cmd; - -------------------------- - -- Report_Not_Installed -- - -------------------------- + ----------------------------- + -- Report_If_Not_Installed -- + ----------------------------- - procedure Report_Not_Installed (Exec : String) is + procedure Report_If_Not_Installed (Exec : String) is begin - if Exec = "gnatstudio" or else Exec = "gnatstudio.exe" then - Reportaise_Command_Failed - ("GNAT Studio not available or not in PATH. " & ASCII.LF & - "You can download it at: " & ASCII.LF & - "https://github.com/AdaCore/gnatstudio/releases"); - elsif Exec = "code" or else Exec = "code.exe" then - Reportaise_Command_Failed - ("VS Code not available or not in PATH. " & ASCII.LF & - "You can download it at: " & ASCII.LF & - "https://code.visualstudio.com/download" & ASCII.LF & - "We also recomend installing the 'AdaCore.ada' extension."); - else - Reportaise_Command_Failed - ("'" & Exec & "' not available or not in PATH."); + if Alire.OS_Lib.Subprocess.Locate_In_Path (Exec) = "" then + -- Executable not in PATH, report an error + if Exec = "gnatstudio" or else Exec = "gnatstudio.exe" then + Reportaise_Command_Failed + ("GNAT Studio not available or not in PATH. " & ASCII.LF & + "You can download it at: " & ASCII.LF & + "https://github.com/AdaCore/gnatstudio/releases"); + elsif Exec = "code" or else Exec = "code.exe" then + Reportaise_Command_Failed + ("VS Code not available or not in PATH. " & ASCII.LF & + "You can download it at: " & ASCII.LF & + "https://code.visualstudio.com/download" & ASCII.LF & + "We also recomend installing the 'AdaCore.ada' extension."); + else + Reportaise_Command_Failed + ("'" & Exec & "' not available or not in PATH."); + end if; end if; - end Report_Not_Installed; + end Report_If_Not_Installed; ------------------ -- Query_Editor -- @@ -132,36 +136,6 @@ package body Alr.Commands.Edit is end; end Query_Editor; - ------------------ - -- Start_Editor -- - ------------------ - - procedure Start_Editor (Root : in out Alire.Roots.Root; - Args : in out AAA.Strings.Vector; - Prj : Relative_Path) - is - Cmd : constant String := Args.First_Element; - - Replaced_Args : AAA.Strings.Vector; - begin - - Args.Delete_First; - - for Elt of Args loop - - -- Replace pattern in Elt, if any - Replaced_Args.Append - (Format.Format - (Elt, - Format.For_Editor (Root, Prj), - Is_Path => True)); - end loop; - - Trace.Info ("Editing crate with: ['" & Cmd & "' '" & - AAA.Strings.Flatten (Replaced_Args, "', '") & "']"); - Alire.OS_Lib.Subprocess.Checked_Spawn (Cmd, Replaced_Args); - end Start_Editor; - ------------- -- Execute -- ------------- @@ -203,29 +177,19 @@ package body Alr.Commands.Edit is Cmd.Root.Export_Build_Environment; declare - Editor_Cmd : constant String := - Builtins.Editor_Cmd.Get; - - Edit_Args : AAA.Strings.Vector := - AAA.Strings.Split (Editor_Cmd, ' '); - - Exec : constant String := Edit_Args.First_Element; - Project_Files : constant AAA.Strings.Vector := Cmd.Root.Release.Project_Files (Platforms.Current.Properties, With_Path => True); begin - if Alire.OS_Lib.Subprocess.Locate_In_Path (Exec) = "" then - -- Executable not in PATH, report an error - Report_Not_Installed (Exec); - end if; - if Project_Files.Length = 0 then Reportaise_Command_Failed ("No project file to open for this crate."); elsif Project_Files.Length = 1 then - Start_Editor (Cmd.Root, Edit_Args, Project_Files.First_Element); + Spawn.Settings_Command + (Builtins.Editor_Cmd.Get, + Format.For_Editor (Cmd.Root, Project_Files.First_Element), + Report_If_Not_Installed'Access); elsif Cmd.Prj = null or else @@ -240,7 +204,10 @@ package body Alr.Commands.Edit is ("Please specify a project file with --project=."); else - Start_Editor (Cmd.Root, Edit_Args, Cmd.Prj.all); + Spawn.Settings_Command + (Builtins.Editor_Cmd.Get, + Format.For_Editor (Cmd.Root, Cmd.Prj.all), + Report_If_Not_Installed'Access); end if; end; end Execute; diff --git a/src/alr/alr-commands-settings.adb b/src/alr/alr-commands-settings.adb index 5043460d..95e4b4e3 100644 --- a/src/alr/alr-commands-settings.adb +++ b/src/alr/alr-commands-settings.adb @@ -112,12 +112,12 @@ package body Alr.Commands.Settings is Alire.Settings.Edit.Set_Boolean (Lvl, Key, Boolean'Value (Val), - Check => Alire.Settings.Edit.Valid_Builtin'Access); + Check => Alire.Settings.Edit.Valid_Builtin_Check (Lvl)); else Alire.Settings.Edit.Set (Lvl, Key, Val, - Check => Alire.Settings.Edit.Valid_Builtin'Access); + Check => Alire.Settings.Edit.Valid_Builtin_Check (Lvl)); end if; end; diff --git a/testsuite/drivers/alr.py b/testsuite/drivers/alr.py index c2ac9ae6..da5a847e 100644 --- a/testsuite/drivers/alr.py +++ b/testsuite/drivers/alr.py @@ -640,18 +640,40 @@ def external_compiler_version() -> str: # Capture version return re.search("gnat_external ([0-9.]+)", p.out, re.MULTILINE).group(1) +def alr_settings_unset(key: str, local: bool = False): + """ + Unset a key with `alr settings` + + Sets the value globally unless `local` is `True`. + """ + if local: + run_alr("settings", "--unset", key) + else: + run_alr("settings", "--global", "--unset", key) + +def alr_settings_set(key: str, value: str, local: bool = False): + """ + Set a key-value pair with `alr settings` + + Sets the value globally unless `local` is `True`. + """ + if local: + run_alr("settings", "--set", key, value) + else: + run_alr("settings", "--global", "--set", key, value) + def unselect_compiler(): """ Leave compiler configuration as if "None" was selected by the user in the assistant. """ - run_alr("settings", "--global", "--unset", "toolchain.use.gnat") - run_alr("settings", "--global", "--unset", "toolchain.external.gnat") + alr_settings_unset("toolchain.use.gnat") + alr_settings_unset("toolchain.external.gnat") def set_default_user_settings(): """ Set the default alr settings that are undone by the testsuite defaults """ - run_alr("settings", "--global", "--set", "index.auto_community", "true") - run_alr("settings", "--global", "--set", "toolchain.assistant", "true") + alr_settings_set("index.auto_community", "true") + alr_settings_set("toolchain.assistant", "true") diff --git a/testsuite/drivers/helpers.py b/testsuite/drivers/helpers.py index b4679d6b..53c8d953 100644 --- a/testsuite/drivers/helpers.py +++ b/testsuite/drivers/helpers.py @@ -10,6 +10,7 @@ import shutil import stat import sys from subprocess import run +from typing import Union from zipfile import ZipFile @@ -314,7 +315,103 @@ class FileLock(): self.lock_file.close() -GIT_WRAPPER_TEMPLATE = """\ +MOCK_COMMAND_TEMPLATE = """\ +with Ada.Command_Line; +with GNAT.OS_Lib; + +procedure Main is + Num_Args : constant Integer := Ada.Command_Line.Argument_Count; + Arg_List : GNAT.OS_Lib.Argument_List (1 .. Num_Args + 1); + Exit_Code : Integer; +begin + -- Set arguments to pass to 'python' (the path to the script, followed by + -- the arguments to pass thereto). + Arg_List (1) := new String'("{script_path}"); + for I in 1 .. Num_Args loop + Arg_List (I + 1) := new String'(Ada.Command_Line.Argument (I)); + end loop; + -- Run the Python script, passing the output directly to stdout and stderr. + Exit_Code := + GNAT.OS_Lib.Spawn + (Program_Name => GNAT.OS_Lib.Locate_Exec_On_Path ("python").all, + Args => Arg_List); + -- Imitate the script's exit status. + Ada.Command_Line.Set_Exit_Status (Ada.Command_Line.Exit_Status (Exit_Code)); +end Main; +""" + +class MockCommand: + """ + Replace a command with a Python script. + + Can be used as a context manager or via the `enable()` and `disable()` + methods. + + The mock command is placed under the path `dir`, which is temporarily + prepended to `PATH`. Changes to `PATH` are overridden in the case of tools + installed by MSYS2, so a `MockCommand` for such a tool will be ignored + unless `msys2.install_dir` is also set to an empty directory. + + `dir` should be empty or non-existent, except that it may also be used by + other instances of `MockCommand` with different `name`s. + """ + + def __init__(self, name: str, script: str, dir: Union[str, os.PathLike]): + self._name = name + self._script = script + self._dir = os.path.realpath(dir) + + def __enter__(self): + self.enable() + + def __exit__(self, type, value, traceback): + self.disable() + + def enable(self): + """ + Enable mocking for the command. + """ + os.makedirs(self._dir, exist_ok=True) + # Write the script to the directory + self._script_dir = os.path.join(self._dir, "scripts") + self._script_path = os.path.join(self._script_dir, self._name) + os.makedirs(self._script_dir, exist_ok=True) + with open(self._script_path, "x") as f: + f.write(self._script) + # Only binary executables are consistently recognised on the `PATH` in + # Windows, so we need to compile a binary wrapper for the script. + build_dir = os.path.join(self._dir, "build") + os.makedirs(build_dir) + with open(os.path.join(build_dir, "main.adb"), "x") as f: + f.write(MOCK_COMMAND_TEMPLATE.format(script_path=self._script_path)) + run( + ["gnat", "make", "-q", os.path.join(build_dir, "main.adb")], + cwd=build_dir + ).check_returncode() + # Copy the binary to a directory on PATH + suffix = ".exe" if on_windows() else "" + self._bin_dir = os.path.join(self._dir, "path_dir") + self._bin_path = os.path.join(self._bin_dir, self._name + suffix) + os.makedirs(self._bin_dir, exist_ok=True) + shutil.copy(os.path.join(build_dir, "main" + suffix), self._bin_path) + shutil.rmtree(build_dir) + os.environ["PATH"] = f'{self._bin_dir}{os.pathsep}{os.environ["PATH"]}' + + def disable(self): + """ + Disable mocking for the command. + """ + # Restore PATH + os.environ["PATH"] = os.environ["PATH"].replace( + f'{self._bin_dir}{os.pathsep}', '', 1 + ) + # Delete the script and binary + os.remove(self._bin_path) + os.remove(self._script_path) + + + +SUBSTITUTION_WRAPPER_TEMPLATE = """\ #! /usr/bin/env python import subprocess, sys substitution_dict = {substitution_dict} @@ -322,8 +419,8 @@ substitution_dict = {substitution_dict} args = sys.argv[1:] for key in substitution_dict: args = [arg.replace(key, substitution_dict[key]) for arg in args] -# Run git -p = subprocess.run(['{actual_git_path}'] + args, capture_output=True) +# Run the command +p = subprocess.run([r'{actual_cmd_path}'] + args, capture_output=True) # Output substitutions stdout, stderr = p.stdout.decode(), p.stderr.decode() for key in substitution_dict: @@ -335,57 +432,23 @@ print(stderr, file=sys.stderr, end="") sys.exit(p.returncode) """ -class MockGit: +class WrapCommand(MockCommand): """ - NON-WINDOWS-ONLY - A context manager which mocks the git command with string substitutions. + Wrap the command `name` with string substitutions applied to its arguments + and output. - The string substitutions are specified by the dictionary substitution_dict. - Every non-overlapping occurrence of each of its keys in a command line - argument is replaced with its corresponding value before being passed to - git. The reverse substitution is applied to git's output. The substitutions - are applied in the order in which they appear in substitution_dict. + The string substitutions are specified by the dictionary `subs`. Every + non-overlapping occurrence of each of its keys in a command line argument is + replaced with its corresponding value before being passed to the command. + The reverse substitution is applied to the command's output. The + substitutions are applied in the order in which they appear in `subs`. - The mocked version of git will be placed in mock_git_dir, which will be - temporarily added to PATH. + The other arguments are the same as for `MockCommand`. """ - def __init__(self, substitution_dict, mock_git_dir): - self._substitution_dict = substitution_dict - self._mock_git_dir = mock_git_dir - - def __enter__(self): - # Mocking on Windows would require git.exe wrapper - if on_windows(): - print('SKIP: git mocking unavailable on Windows') - sys.exit(0) - - # Create a wrapper script for git - wrapper_script = GIT_WRAPPER_TEMPLATE.format( - substitution_dict=self._substitution_dict, - actual_git_path=shutil.which("git") - ) - # Add the directory to PATH - try: - os.mkdir(self._mock_git_dir) - except FileExistsError: - pass - os.environ["PATH"] = ( - f'{self._mock_git_dir}{os.pathsep}{os.environ["PATH"]}' - ) - # Write the script to the directory - wrapper_descriptor = os.open( - os.path.join(self._mock_git_dir, "git"), - flags=(os.O_WRONLY | os.O_CREAT | os.O_EXCL), - mode=0o764, - ) - with open(wrapper_descriptor, "w") as f: - f.write(wrapper_script) - - def __exit__(self, type, value, traceback): - # Restore PATH - os.environ["PATH"] = os.environ["PATH"].replace( - f'{self._mock_git_dir}{os.pathsep}', '', 1 + def __init__(self, name: str, subs: dict[str:str], dir: Union[str, os.PathLike]): + wrapper_script = SUBSTITUTION_WRAPPER_TEMPLATE.format( + substitution_dict=subs, + actual_cmd_path=shutil.which(name) ) - # Delete the wrapper script - os.remove(os.path.join(self._mock_git_dir, "git")) + super().__init__(name, wrapper_script, dir) diff --git a/testsuite/tests/get/custom-download-command/my_index/index/index.toml b/testsuite/tests/get/custom-download-command/my_index/index/index.toml new file mode 100644 index 00000000..bad265e4 --- /dev/null +++ b/testsuite/tests/get/custom-download-command/my_index/index/index.toml @@ -0,0 +1 @@ +version = "1.1" diff --git a/testsuite/tests/get/custom-download-command/my_index/index/li/libhello/libhello-1.0.0.toml b/testsuite/tests/get/custom-download-command/my_index/index/li/libhello/libhello-1.0.0.toml new file mode 100644 index 00000000..7dd1eb7b --- /dev/null +++ b/testsuite/tests/get/custom-download-command/my_index/index/li/libhello/libhello-1.0.0.toml @@ -0,0 +1,10 @@ +description = "\"Hello, world!\" demonstration project support library" +name = "libhello" +version = "1.0.0" +maintainers = ["alejandro@mosteo.com"] + +[origin] +# Since we are mocking the download script, the url doesn't matter, as long as +# it is recognised as a remote source archive. +url = "https://some.host/path/to/archive.tgz" +hashes = ["sha512:99fa3a55540d0655c87605b54af732f76a8a363015f183b06e98aa91e54c0e69397872718c5c16f436dd6de0fba506dc50c66d34a0e5c61fb63cb01fa22f35ac"] diff --git a/testsuite/tests/get/custom-download-command/test.py b/testsuite/tests/get/custom-download-command/test.py new file mode 100644 index 00000000..aadf50f7 --- /dev/null +++ b/testsuite/tests/get/custom-download-command/test.py @@ -0,0 +1,93 @@ +""" +Test `get`ing a tarball release with a different download command configured. +""" + + +import os +import shutil + +from drivers.alr import crate_dirname, fixtures_path, run_alr, alr_settings_set +from drivers.asserts import assert_match, assert_substring +from drivers.helpers import MockCommand + + +# The script for the mock download command. Prints its arguments, then copies +# `libhello_1.0.0.tgz` from fixtures to the specified output file. +COMMAND_SCRIPT = f"""\ +import shutil, sys +print(f"Mock command called with arguments: {{sys.argv}}") +shutil.copy(r"{fixtures_path('crates', 'libhello_1.0.0.tgz')}", sys.argv[4]) +""" + + +def set_download_cmd(cmd: str): + """ + Set the download command in `alr`'s global settings to `cmd`. + """ + alr_settings_set("origins.archive.download_cmd", cmd) + + +# Mock `curl` so it always fails, and put the mock download command on `PATH`. +alr_settings_set("msys2.install_dir", os.path.abspath("does_not_exist")) +mock_curl = MockCommand("curl", "raise Exception", "cmd_dir") +mock_download_cmd = MockCommand("command_name", COMMAND_SCRIPT, "cmd_dir") +with mock_curl, mock_download_cmd: + # Verify that `alr get libhello` fails, because it attempts to call `curl`. + p = run_alr("get", "libhello", complain_on_error=False) + assert_match( + ( + r'.*Command \["curl", "https://some\.host/path/to/archive\.tgz",' + r' "-L", "-s", "-o", "[^"]*archive.tgz"\] exited with code 1' + ), + p.out + ) + + # Check that the default curl command is changed correctly when `-v` is + # specified. + p = run_alr("-v", "get", "libhello", complain_on_error=False, quiet=False) + assert_match( + ( + r'.*Command \["curl", "https://some\.host/path/to/archive\.tgz",' + r' "-L", "--progress-bar", "-o", "[^"]*archive.tgz"\] exited with ' + r'code 1' + ), + p.out + ) + + # Configure `alr` to use the mock download command instead of `curl`. + set_download_cmd("command_name arg1 --url=${URL} arg2 ${DEST} arg3") + + # Verify that `alr get libhello` now succeeds, by calling the configured + # command. + pattern = ( + r".*Mock command called with arguments: \['[^']*', 'arg1', " + r"'--url=https://some\.host/path/to/archive\.tgz', 'arg2', " + r"'[^']*archive.tgz', 'arg3'\]" + ) + p = run_alr("get", "libhello", quiet=False) + assert_match(pattern, p.out) + shutil.rmtree(crate_dirname("libhello")) + + # Verify that changing the logging level doesn't affect the command used. + p = run_alr("-v", "get", "libhello", quiet=False) + assert_match(pattern, p.out) + shutil.rmtree(crate_dirname("libhello")) + + # Verify that replacements are case-insensitive. + set_download_cmd("command_name arg1 --url=${UrL} arg2 ${DeSt} arg3") + p = run_alr("get", "libhello", quiet=False) + assert_match(pattern, p.out) + shutil.rmtree(crate_dirname("libhello")) + + # Verify that an invalid formatting key raises an appropriate error. + set_download_cmd("command_name --url=${URL} ${DEST} ${INVALID}") + p = run_alr("get", "libhello", complain_on_error=False) + assert_match(r".*Unknown formatting key: INVALID", p.out) + + # Verify that a command not found on `PATH` raises an appropriate error. + set_download_cmd("non_existent --url=${URL} ${DEST} arg3") + p = run_alr("get", "libhello", complain_on_error=False) + assert_substring("'non_existent' not available or not in PATH.", p.out) + + +print('SUCCESS') diff --git a/testsuite/tests/get/custom-download-command/test.yaml b/testsuite/tests/get/custom-download-command/test.yaml new file mode 100644 index 00000000..0a859639 --- /dev/null +++ b/testsuite/tests/get/custom-download-command/test.yaml @@ -0,0 +1,4 @@ +driver: python-script +indexes: + my_index: + in_fixtures: false diff --git a/testsuite/tests/pin/branch-remote-protocols/test.py b/testsuite/tests/pin/branch-remote-protocols/test.py index 97d2fee2..45e64ca0 100644 --- a/testsuite/tests/pin/branch-remote-protocols/test.py +++ b/testsuite/tests/pin/branch-remote-protocols/test.py @@ -7,7 +7,7 @@ import shutil import subprocess from drivers.alr import alr_pin, alr_unpin, init_local_crate -from drivers.helpers import init_git_repo, git_branch, MockGit +from drivers.helpers import init_git_repo, git_blast, git_branch, WrapCommand from drivers.asserts import assert_eq @@ -58,15 +58,15 @@ mocked_git_dir = os.path.join(os.getcwd(), "mock_path") for url, s_url in zip(urls, sanitised_urls): # Mock git with a wrapper that naively converts the url into the local path # to the "remote" crate. - with MockGit({s_url: remote_path}, mocked_git_dir): + with WrapCommand("git", {s_url: remote_path}, mocked_git_dir): # Create an empty crate, and pin the default branch of the test repo init_local_crate() alr_pin("remote", url=url, branch=default_branch) with open(cache_test_file_path) as f: assert_eq("This is the main branch.\n", f.read()) - # Edit pin to point to the other branch, and verify the cached copy changes - # as it should + # Edit pin to point to the other branch, and verify the cached copy + # changes as it should alr_unpin("remote", update=False) alr_pin("remote", url=url, branch="other") with open(cache_test_file_path) as f: @@ -74,7 +74,7 @@ for url, s_url in zip(urls, sanitised_urls): # Clean up for next test os.chdir("..") - shutil.rmtree("xxx") + git_blast("xxx") print("SUCCESS") diff --git a/testsuite/tests/publish/private-indexes/test.py b/testsuite/tests/publish/private-indexes/test.py index 05788774..736bafed 100644 --- a/testsuite/tests/publish/private-indexes/test.py +++ b/testsuite/tests/publish/private-indexes/test.py @@ -9,7 +9,7 @@ import shutil import subprocess from drivers.alr import run_alr, run_alr_interactive -from drivers.helpers import init_git_repo, MockGit +from drivers.helpers import init_git_repo, WrapCommand from drivers.asserts import assert_match, assert_file_exists @@ -77,7 +77,7 @@ def test( # Mock git with a wrapper that naively converts the url into the local path # to the "remote" crate. mocked_git_dir = os.path.abspath(os.path.join("..", "..", "mocked_git")) - with MockGit({url: remote_path}, mocked_git_dir): + with WrapCommand("git", {url: remote_path}, mocked_git_dir): # Create a "local" clone of the "remote" local_path = os.path.abspath(os.path.join("..", "..", "local", "xxx")) os.makedirs(local_path) diff --git a/testsuite/tests/settings/basics/test.py b/testsuite/tests/settings/basics/test.py index 8c57e87d..8f540da8 100644 --- a/testsuite/tests/settings/basics/test.py +++ b/testsuite/tests/settings/basics/test.py @@ -7,6 +7,7 @@ import os from glob import glob from drivers.alr import run_alr +from drivers.asserts import assert_substring def invalid_key(*args): print("Running: alr settings %s" % " ".join([item for item in args])) @@ -35,6 +36,16 @@ def check_value(key, expected_value, local=True): get = run_alr('settings', '--global', '--get', key) assert get.out == expected_value + "\n", "Got '%s'" % get.out +def check_undefined(key, local=True): + if local: + get = run_alr('settings', '--get', key, complain_on_error=False) + else: + get = run_alr( + 'settings', '--global', '--get', key, complain_on_error=False + ) + assert f"Setting key '{key}' is not defined" in get.out, \ + "Missing error message in: '%s" % get.out + def set_get_unset(key, value, image=None): if image is None: @@ -75,7 +86,11 @@ invalid_key('--get', '--global', '^') # invalid builtins # #################### invalid_builtin('--set', '--global', 'user.github_login', 'This is not a valid login') +check_undefined('user.github_login', local=False) invalid_builtin('--set', '--global', 'user.email', '@ This is not @ valid email address@') +check_undefined('user.email', local=False) +invalid_builtin('--set', '--global', 'distribution.override', 'invalid distribution') +check_undefined('distribution.override', local=False) ############################### # Global Set, Get, Unset, Get # @@ -109,6 +124,11 @@ check_value('test.override', 'is_local') run_alr('settings', '--set', '--global', 'test.override', '"is_global"') check_value('test.override', 'is_local') +# Try setting a Global_Only value and check that it doesn't work +p = run_alr('settings', '--set', 'editor.cmd', 'value', complain_on_error=False) +assert_substring("Configuration key 'editor.cmd' must be set globally", p.out) +check_undefined('editor.cmd') + # Leave the crate context (local keys are not available anymore) os.chdir('..') diff --git a/testsuite/tests/workflows/air-gapping/my_index/crates/hello.tar b/testsuite/tests/workflows/air-gapping/my_index/crates/hello.tar new file mode 100644 index 0000000000000000000000000000000000000000..f017d9a93963de005c453f855a4f71c4cd564e6e GIT binary patch literal 10240 zcmeH}J8y$96or}nD=emVNiYvHuy*LsszZm45f3*IQHTsEf4{bY5}_!`5UNNy9x!mP zANr2Zg>bB)qkkjQ1SFe`j@vd zvCUU4%Pbm~oYE}Qw^^#k~S5iRDE8vonnPyqB1 zu=^^o6KQ)?1C;x}Ye_%rPNrh>QpkpOji67q`gO3MBV8SI{3FrhblimPI@ziDhTwZ# zRKuwLz^RXo9(!CtP(tPk;TZ%#00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`; NKmY_l00gch@C$Dan9KkG literal 0 HcmV?d00001 diff --git a/testsuite/tests/workflows/air-gapping/my_index/crates/hello.tgz b/testsuite/tests/workflows/air-gapping/my_index/crates/hello.tgz deleted file mode 100644 index a39010d27cf1902c1153f4a9ddc7cd3d7570ed4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 334 zcmV-U0kQrciwFP!000001MStlZi6rs1z=`Bg~ik^DK=mR)(#z7b?DGB;>Qg{6e0u4 z>(@4fMp4@+RY+9zd<%$sEkK8B3RzXnoii&>i=xmRSri82QP}GvhXo=EI2XRqeasWO zjfV3+)}U{@L{aLzXUX=bzGL-2s<8RDDx2{eI4S>->HJyX3*>(ZpZVLfMUtmyzlRA} zh~V4&850Nj``n8h>Ye>bzVOfU-)*`>wIXA$+h1GPBqN`km_oqX1T|)`RAM+%s&r8%>PB$`FE?`VCzr79d?1W z%(}fjKu`Z$qv%W1t4uEMOV!f8mGs8dzYg~{-qoK|v(D6Q*=^FbI~AXjvL07WKO3`l ge+R)90ssI20000000000008hiA3Kd7F90Y20A9796#xJL diff --git a/testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tar b/testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tar new file mode 100644 index 0000000000000000000000000000000000000000..6b7091eb10d29b4923ce9fcf73d644e4f71ac703 GIT binary patch literal 10240 zcmeH~U2DQH6oz}c%$h59nNn4!%ekZl@qopWV2eOl! zn4FUspEvIbCUG!hNsopyk>pBW_hU?f>Pem`AV%XHRO_RF1uIPqgS++u)>m6yo zbCKmfCq!9<{(3*#{B$bbTyYaKiPS&i;kE975cypFDYZa<6D|Pi{}&hNNPjsP_)*s3 zmka$VmA;q#f&X2)^Z!hIf9>!kmn&{{hnHUnwW=X~9G0J;gjLjoH)Sk*J&|;*&ru74b5xrNLZ-`T;%^ zQP`3DeB-_P3$~4ye>5SZmatn7cTubtCYHy6;h8@K(~tlXKmter2_OL^fCP{L5&*IQ>BY literal 0 HcmV?d00001 diff --git a/testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tgz b/testsuite/tests/workflows/air-gapping/my_index/crates/libhello.tgz deleted file mode 100644 index c5882e61a64bcdbdd93a0e082d77f0fedb4cad96..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 428 zcmV;d0aN}TiwFP!000001MQVhPlGTR#yRsTUOCM~=YUcUoJ>4uvN#h>Jdy%+W3sUn z$lSNLWk4JYE{kl&`2H@y`=@R4^m$3*V8)Xqz0pp`NaDJVdYZ07swXYu6`Dz?ODv1J z@_l5QwoNo}PI#tJA*EKc%+R-;x&q^*;wY`qv*r z7G*u|p#-LsbnEZZmi~lTrUs)P&(O&~*MG&rcQ)lvOAm3T>j1D4X~?5OaJW}t|1}j! zc#X|&m! z3)gM7!LL&S>R`LFbFQexp`^(8?rCH!;L~_n<)4b&AID1$1EcJ}g0))U!niC;)pWvn zbqtOF8}Cn7qUSvh`QIW|%l|ed6#ah=q#x%z%wgQ}`CBUBIW0uU{kvFXr7;JF{7F