From 2c2b26926a430ff2cdb87f0946094660497eae77 Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9sar=20Sagaert?= Date: Mon, 17 Mar 2025 13:00:38 +0100 Subject: [PATCH] docs: test command usage (#1888) * docs: document the manifest [test] section and the new alr test command * move 'success' log to end of test runs * improve help and diagnostics in alr test * address review comments * changelog and breaking changes * improve getting started section clarity * address comments --- BREAKING.md | 5 ++- doc/catalog-format-spec.md | 42 ++++++++++++++++++++++--- doc/getting-started.md | 54 +++++++++++++++++++++++++++++++-- doc/user-changes.md | 26 +++++++++++++++- src/alire/alire-test_runner.adb | 2 -- src/alr/alr-commands-test.adb | 23 +++++++++++--- src/alr/alr-commands-test.ads | 7 ++++- 7 files changed, 142 insertions(+), 17 deletions(-) diff --git a/BREAKING.md b/BREAKING.md index 9fb6bf6d..3ca9d289 100644 --- a/BREAKING.md +++ b/BREAKING.md @@ -1,10 +1,13 @@ Log of breaking changes in index or alr. Future versions are as of today and may change. -### alr 3.0.0 + index 1.4.0 +### alr 3.0.0 + index 1.5.0 - alr: removed `ALR_CONFIG` environment variable. - alr: removed `alr config` command. +- alr: removed remote testing in `alr test` command. +- alr: added new behavior to `alr test` and a built-in test runner. +- manifest: added `[test]` section. ### We are here diff --git a/doc/catalog-format-spec.md b/doc/catalog-format-spec.md index 0107a146..bd901eb4 100644 --- a/doc/catalog-format-spec.md +++ b/doc/catalog-format-spec.md @@ -396,11 +396,14 @@ static, i.e. they cannot depend on the context. successfully completed. This kind of action is run for all releases in the solution. - - `test`: the command is run on demand for crate testing within the Alire - ecosystem (using `alr index-test`). This kind of action is run only for the - root crate being tested. The crate is not built beforehand when a test - action is defined so, if a build is necessary, it should be explicitly - given as part of the action sequence. + - _[⚠️ deprecated]_ `test`: the command is run on demand for crate testing + within the Alire ecosystem (using `alr test`). This kind of action is run + only for the root crate being tested. The crate is not built beforehand + when a test action is defined so, if a build is necessary, it should be + explicitly given as part of the action sequence. + + *Test actions are deprecated; use the `[test]` section in the manifest to + configure crate tests instead.* Since actions may end being run more than once they should take this into account and allow multiple runs with the expected results intended by the @@ -435,6 +438,35 @@ static, i.e. they cannot depend on the context. # Another action, that needs not be also conditional (but could be). ``` + - `test`: optional section that configures the behavior of `alr test`. + The test section accepts dynamic expressions, making it possible to use + different test runners on different platforms. The general syntax for this + section is: + + ```toml + [test] + runner = + # OR + command = ["", "", ...] + + directory = + jobs = + ``` + + `` is a built-in test runner name, currently only `'alire'`. + `` is an array of strings that will be run as an external command. + + When running `alr test`, Alire will first enter the given `directory`, a + subfolder of the crate root, and execute either the built-in test runner or + the given external command. + + The `jobs` parameter describes how many tests should be run in parallel. It + is only valid in the context of the built-in test runner for now. + + The `[test]` section also accepts an array of test runners (with + `[[test]]`), although you lose the ability to pass extra arguments to the + test command with `alr test`. + - `auto-gpr-with`: optional Boolean value that specifies if the project (gpr) files of a crate can be automatically depended upon ('withed') directly by the root project file. (The default is true.) This feature is meant to simplify the process diff --git a/doc/getting-started.md b/doc/getting-started.md index e10f0e59..6233db8e 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -305,12 +305,62 @@ pristine settings](settings#relocating-your-settings) ## Running tests +### Self-tests + `alr` comes with a test suite for self-checks. See the instructions in the [README](https://github.com/alire-project/alire/blob/master/testsuite/README.md) of the `testsuite` folder. -Additionally, you can test in batch the building of crates in your platform -with the `alr index-test` command. (See `alr index-test --help` output for instructions.) +### Testing your crates + +Alire provides an `alr test` command for testing the functionality of your +projects. It needs to be configured in a `[test]` section of the Alire manifest, +by specifying either `runner = 'alire'` or a custom runner with +`command = [...]`. +Any arguments passed to the `alr test` command are passed as-is to the runner +command (or the built-in runner, which uses them for test filtering). + +It is recommended to provide tests for a library in a `tests` or `testsuite` +subfolder of the root crate; if your test suite is an Alire crate, it can then +depend on the root crate by using a pinned dependency, like so: + +```toml +[[depends-on]] +mycrate = '*' + +[[pins]] +mycrate = { path = '..' } +``` + +### Built-in runner + +The built-in Alire test runner provides a simple way to run `.adb` files as +separate tests. It requires setting up a subcrate, with a GPR file that defines +its executables in a specific way: + +```ada +with "config/_list_config.gpr" +-- ... +for Main use _List_Config.Test_Files; +``` + +This subcrate is automatically generated by `alr init` (unless one specifies +`--no-test`), so you will not have to write all of this by hand. + +When running the `alr test` command, it will put every `.adb` file in `/src` in +the generated `config/_list_config.gpr` file, then build them all +and run them in parallel in accordance with the `jobs` parameter (either from +the manifest or the command line). + +Support code for the tests can be placed into another folder, like `/common`; +only `.adb` files in `/src` are counted as tests. Each test must be a main +procedure, taking no arguments and returning a non-zero error code on failure +(for instance, by raising an exception, either manually or through some kind of +`Assert` function). + +On execution, tests can be filtered by providing additional arguments to +`alr test`: using `alr test abcd efgh` will only run tests whose name contain +either 'abcd' or 'efgh'. ## Migration of an existing Ada/SPARK project to Alire diff --git a/doc/user-changes.md b/doc/user-changes.md index 3bd6287c..fd26a630 100644 --- a/doc/user-changes.md +++ b/doc/user-changes.md @@ -4,7 +4,31 @@ This document is a development diary summarizing changes in `alr` that notably affect the user experience. It is intended as a one-stop point for users to stay on top of `alr` new features. -## Release `2.2` +## Release `3.0` + +### New test command + +PR [#1874](https://github.com/alire-project/alire/pull/1874) + +The `alr test` command can now be configured with a new `[test]` section in the +Alire manifest, with the option to use a built-in test runner or an external +command. + +```toml +[test] +runner = 'alire' +# OR +command = ["my", "custom", "runner"] +``` + +The built-in test runner allows crate authors to compile and run tests from +simple `.adb` files. It compiles and runs every `.adb` file of the subcrate's +`/src` folder as a separate test. A skeleton test subcrate is now generated +with `alr init` (this can be prevented with the `--no-test` flag). + +For backwards compatibility, running `alr test` without a `[test]` section in +the manifest will still run local test actions, but they should be considered +deprecated. The remote testing capabilities of `alr test` have been removed. ## Release `2.1` diff --git a/src/alire/alire-test_runner.adb b/src/alire/alire-test_runner.adb index a8962ad1..74ef2276 100644 --- a/src/alire/alire-test_runner.adb +++ b/src/alire/alire-test_runner.adb @@ -282,8 +282,6 @@ package body Alire.Test_Runner is Ada.Text_IO.Flush; if Driver.Fail_Count /= 0 then Trace.Error ("failed" & Driver.Fail_Count'Image & " tests"); - else - Alire.Put_Success ("Test run completed successfully"); end if; return Driver.Fail_Count; else diff --git a/src/alr/alr-commands-test.adb b/src/alr/alr-commands-test.adb index e735676b..4e4ab976 100644 --- a/src/alr/alr-commands-test.adb +++ b/src/alr/alr-commands-test.adb @@ -72,12 +72,24 @@ package body Alr.Commands.Test is Execute_Legacy (Cmd.Root); end if; - if not Args.Is_Empty and then All_Settings.Length > 1 then + if not Args.Is_Empty + and then (Cmd.Jobs >= 0 or else All_Settings.Length > 1) + then Trace.Warning ("arguments cannot be forwarded to test runners when several " & "exist."); end if; + if All_Settings.Length = 1 + and then Settings (All_Settings.First_Element).Runner.Kind = External + and then Cmd.Jobs >= 0 + then + Trace.Warning + ("the --jobs flag is not forwarded to external commands. If you " + & "intended to pass it to an external test runner, put it after " + & """--"" in the command line."); + end if; + for Test_Setting of All_Settings loop if Alire.Directories.Is_Directory (+Settings (Test_Setting).Directory) then @@ -102,7 +114,7 @@ package body Alr.Commands.Test is Cmd.Optional_Root.Discard; if All_Settings.Length > 1 then - Trace.Info ("running test with" & S.Image); + Alire.Put_Info ("running test with" & S.Image); end if; case S.Runner.Kind is @@ -113,7 +125,7 @@ package body Alr.Commands.Test is Alire.Test_Runner.Run (Cmd.Root, Get_Args, - (if Cmd.Jobs = -1 then S.Jobs else Cmd.Jobs)); + (if Cmd.Jobs < 0 then S.Jobs else Cmd.Jobs)); when External => Failures := @@ -138,6 +150,8 @@ package body Alr.Commands.Test is & "' does not exist."); end if; end loop; + + Alire.Put_Success ("Successful test run"); end Execute; ---------------------- @@ -147,8 +161,7 @@ package body Alr.Commands.Test is overriding function Long_Description (Cmd : Command) return AAA.Strings.Vector is (AAA.Strings.Empty_Vector.Append - ("Run the test runner defined in the manifest, " - & "or the builtin test runner") + ("Run the test runner as defined in the manifest.") .Append ("") .Append ("The builtin test runner takes an extra --jobs parameter, " diff --git a/src/alr/alr-commands-test.ads b/src/alr/alr-commands-test.ads index 8074c5aa..819590ac 100644 --- a/src/alr/alr-commands-test.ads +++ b/src/alr/alr-commands-test.ads @@ -25,7 +25,12 @@ package Alr.Commands.Test is overriding function Usage_Custom_Parameters (Cmd : Command) return String - is ("[test_names]..."); + is ("[args]..."); + + overriding + function Switch_Parsing + (Cmd : Command) return CLIC.Subcommand.Switch_Parsing_Kind + is (CLIC.Subcommand.Before_Double_Dash); private -- 2.39.5