NixOS/nixpkgs
1
0
Fork 1
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-10 18:12:34 +09:00
Code Activity

nixos/testing: Move entrypoint to nixos/lib + doc

Browse source
This commit is contained in:
Robert Hensing 2022-06-15 16:59:21 +02:00
parent 5727fd3e6f
commit b0c781cc41
5 changed files with 256 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

73
nixos/doc/manual/development/writing-nixos-tests.section.md
View file

@ -1,9 +1,9 @@
# Writing Tests {#sec-writing-nixos-tests} # Writing Tests {#sec-writing-nixos-tests}
A NixOS test is a Nix expression that has the following structure: A NixOS test is a module that has the following structure:
```nix ```nix
import ./make-test-python.nix { {
# One or more machines: # One or more machines:
nodes = nodes =
@ -21,7 +21,10 @@ import ./make-test-python.nix {
} }
``` ```
The attribute `testScript` is a bit of Python code that executes the We refer to the whole test above as a test module, whereas the values
in `nodes.<name>` are NixOS modules. (A NixOS configuration is a module.)
The option `testScript` is a bit of Python code that executes the
test (described below). During the test, it will start one or more test (described below). During the test, it will start one or more
virtual machines, the configuration of which is described by virtual machines, the configuration of which is described by
the attribute `nodes`. the attribute `nodes`.
@ -34,7 +37,64 @@ when switching between consoles, and so on. An interesting multi-node test is
[`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix). [`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix).
It uses two client nodes to test correct locking across server crashes. It uses two client nodes to test correct locking across server crashes.
There are a few special NixOS configuration options for test VMs: ## Calling a test {#sec-calling-nixos-tests}
Tests are invoked a bit differently depending on whether the test lives in NixOS or in another project.
### Testing within NixOS {#sec-call-nixos-test-in-nixos}
Test modules can be instantiated into derivations in multiple ways.
Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix).
```nix
hostname = runTest ./hostname.nix;
```
Overrides can be added by defining an anonymous module in `all-tests.nix`.
For the purpose of constructing a test matrix, use the `matrix` options instead.
```nix
hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
```
You can run a test with attribute name `mytest` in `all-tests.nix` by invoking:
```shell
nix-build -A nixosTests.mytest
```
### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos}
Outside the `nixpkgs` repository, you can instantiate the test by first acquiring the NixOS library,
```nix
# regular nix
let nixos-lib = import (nixpkgs + "/nixos/lib") { };
in
```
```nix
# flake
let nixos-lib = nixpkgs.lib.nixos;
in
```
... and then invoking `runTest`, for example:
```nix
nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
```
`runTest` returns a derivation that runs the test.
## Configuring the nodes {#sec-nixos-test-nodes}
There are a few special NixOS options for test VMs:
`virtualisation.memorySize` `virtualisation.memorySize`
@ -304,7 +364,7 @@ For faster dev cycles it\'s also possible to disable the code-linters
(this shouldn\'t be commited though): (this shouldn\'t be commited though):
```nix ```nix
import ./make-test-python.nix { {
skipLint = true; skipLint = true;
nodes.machine = nodes.machine =
{ config, pkgs, ... }: { config, pkgs, ... }:
@ -336,7 +396,7 @@ Similarly, the type checking of test scripts can be disabled in the following
way: way:
```nix ```nix
import ./make-test-python.nix { {
skipTypeCheck = true; skipTypeCheck = true;
nodes.machine = nodes.machine =
{ config, pkgs, ... }: { config, pkgs, ... }:
@ -400,7 +460,6 @@ added using the parameter `extraPythonPackages`. For example, you could add
`numpy` like this: `numpy` like this:
```nix ```nix
import ./make-test-python.nix
{ {
extraPythonPackages = p: [ p.numpy ]; extraPythonPackages = p: [ p.numpy ];

217
nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
View file

@ -1,10 +1,10 @@
<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests"> <section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests">
<title>Writing Tests</title> <title>Writing Tests</title>
<para> <para>
A NixOS test is a Nix expression that has the following structure: A NixOS test is a module that has the following structure:
</para> </para>
<programlisting language="bash"> <programlisting language="bash">
import ./make-test-python.nix { {
# One or more machines: # One or more machines:
nodes = nodes =
@ -22,7 +22,12 @@ import ./make-test-python.nix {
} }
</programlisting> </programlisting>
<para> <para>
The attribute <literal>testScript</literal> is a bit of Python code We refer to the whole test above as a test module, whereas the
values in <literal>nodes.&lt;name&gt;</literal> are NixOS modules.
(A NixOS configuration is a module.)
</para>
<para>
The option <literal>testScript</literal> is a bit of Python code
that executes the test (described below). During the test, it will that executes the test (described below). During the test, it will
start one or more virtual machines, the configuration of which is start one or more virtual machines, the configuration of which is
described by the attribute <literal>nodes</literal>. described by the attribute <literal>nodes</literal>.
@ -38,78 +43,149 @@ import ./make-test-python.nix {
It uses two client nodes to test correct locking across server It uses two client nodes to test correct locking across server
crashes. crashes.
</para> </para>
<para> <section xml:id="sec-calling-nixos-tests">
There are a few special NixOS configuration options for test VMs: <title>Calling a test</title>
</para> <para>
<variablelist> Tests are invoked a bit differently depending on whether the test
<varlistentry> lives in NixOS or in another project.
<term> </para>
<literal>virtualisation.memorySize</literal> <section xml:id="sec-call-nixos-test-in-nixos">
</term> <title>Testing within NixOS</title>
<listitem> <para>
<para> Test modules can be instantiated into derivations in multiple
The memory of the VM in megabytes. ways.
</para> </para>
</listitem> <para>
</varlistentry> Tests that are part of NixOS are added to
<varlistentry> <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix"><literal>nixos/tests/all-tests.nix</literal></link>.
<term> </para>
<literal>virtualisation.vlans</literal> <programlisting language="bash">
</term> hostname = runTest ./hostname.nix;
<listitem> </programlisting>
<para> <para>
The virtual networks to which the VM is connected. See Overrides can be added by defining an anonymous module in
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link> <literal>all-tests.nix</literal>. For the purpose of
for an example. constructing a test matrix, use the <literal>matrix</literal>
</para> options instead.
</listitem> </para>
</varlistentry> <programlisting language="bash">
<varlistentry> hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
<term> </programlisting>
<literal>virtualisation.writableStore</literal> <para>
</term> You can run a test with attribute name <literal>mytest</literal>
<listitem> in <literal>all-tests.nix</literal> by invoking:
<para> </para>
By default, the Nix store in the VM is not writable. If you <programlisting>
enable this option, a writable union file system is mounted on nix-build -A nixosTests.mytest
top of the Nix store to make it appear writable. This is </programlisting>
necessary for tests that run Nix operations that modify the </section>
store. <section xml:id="sec-call-nixos-test-outside-nixos">
</para> <title>Testing outside the NixOS project</title>
</listitem> <para>
</varlistentry> Outside the <literal>nixpkgs</literal> repository, you can
</variablelist> instantiate the test by first acquiring the NixOS library,
<para> </para>
For more options, see the module <programlisting language="bash">
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>. # regular nix
</para> let nixos-lib = import (nixpkgs + &quot;/nixos/lib&quot;) { };
<para> in
The test script is a sequence of Python statements that perform </programlisting>
various actions, such as starting VMs, executing commands in the <programlisting language="bash">
VMs, and so on. Each virtual machine is represented as an object # flake
stored in the variable <literal>name</literal> if this is also the let nixos-lib = nixpkgs.lib.nixos;
identifier of the machine in the declarative config. If you in
specified a node <literal>nodes.machine</literal>, the following </programlisting>
example starts the machine, waits until it has finished booting, <para>
then executes a command and checks that the output is more-or-less … and then invoking <literal>runTest</literal>, for example:
correct: </para>
</para> <programlisting language="bash">
<programlisting language="python"> nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
</programlisting>
<para>
<literal>runTest</literal> returns a derivation that runs the
test.
</para>
</section>
</section>
<section xml:id="sec-nixos-test-nodes">
<title>Configuring the nodes</title>
<para>
There are a few special NixOS options for test VMs:
</para>
<variablelist>
<varlistentry>
<term>
<literal>virtualisation.memorySize</literal>
</term>
<listitem>
<para>
The memory of the VM in megabytes.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.vlans</literal>
</term>
<listitem>
<para>
The virtual networks to which the VM is connected. See
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link>
for an example.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.writableStore</literal>
</term>
<listitem>
<para>
By default, the Nix store in the VM is not writable. If you
enable this option, a writable union file system is mounted
on top of the Nix store to make it appear writable. This is
necessary for tests that run Nix operations that modify the
store.
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
For more options, see the module
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>.
</para>
<para>
The test script is a sequence of Python statements that perform
various actions, such as starting VMs, executing commands in the
VMs, and so on. Each virtual machine is represented as an object
stored in the variable <literal>name</literal> if this is also the
identifier of the machine in the declarative config. If you
specified a node <literal>nodes.machine</literal>, the following
example starts the machine, waits until it has finished booting,
then executes a command and checks that the output is more-or-less
correct:
</para>
<programlisting language="python">
machine.start() machine.start()
machine.wait_for_unit(&quot;default.target&quot;) machine.wait_for_unit(&quot;default.target&quot;)
if not &quot;Linux&quot; in machine.succeed(&quot;uname&quot;): if not &quot;Linux&quot; in machine.succeed(&quot;uname&quot;):
raise Exception(&quot;Wrong OS&quot;) raise Exception(&quot;Wrong OS&quot;)
</programlisting> </programlisting>
<para> <para>
The first line is technically unnecessary; machines are implicitly The first line is technically unnecessary; machines are implicitly
started when you first execute an action on them (such as started when you first execute an action on them (such as
<literal>wait_for_unit</literal> or <literal>succeed</literal>). If <literal>wait_for_unit</literal> or <literal>succeed</literal>).
you have multiple machines, you can speed up the test by starting If you have multiple machines, you can speed up the test by
them in parallel: starting them in parallel:
</para> </para>
<programlisting language="python"> <programlisting language="python">
start_all() start_all()
</programlisting> </programlisting>
</section>
<section xml:id="ssec-machine-objects"> <section xml:id="ssec-machine-objects">
<title>Machine objects</title> <title>Machine objects</title>
<para> <para>
@ -563,7 +639,7 @@ machine.wait_for_unit(&quot;xautolock.service&quot;, &quot;x-session-user&quot;)
code-linters (this shouldn't be commited though): code-linters (this shouldn't be commited though):
</para> </para>
<programlisting language="bash"> <programlisting language="bash">
import ./make-test-python.nix { {
skipLint = true; skipLint = true;
nodes.machine = nodes.machine =
{ config, pkgs, ... }: { config, pkgs, ... }:
@ -595,7 +671,7 @@ import ./make-test-python.nix {
the following way: the following way:
</para> </para>
<programlisting language="bash"> <programlisting language="bash">
import ./make-test-python.nix { {
skipTypeCheck = true; skipTypeCheck = true;
nodes.machine = nodes.machine =
{ config, pkgs, ... }: { config, pkgs, ... }:
@ -669,7 +745,6 @@ def foo_running():
<literal>numpy</literal> like this: <literal>numpy</literal> like this:
</para> </para>
<programlisting language="bash"> <programlisting language="bash">
import ./make-test-python.nix
{ {
extraPythonPackages = p: [ p.numpy ]; extraPythonPackages = p: [ p.numpy ];

8
nixos/lib/default.nix
View file

@ -21,6 +21,8 @@ let
seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v); seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v);
eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; }; eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; };
testing-lib = import ./testing/default.nix { inherit lib; };
in in
/* /*
This attribute set appears as lib.nixos in the flake, or can be imported This attribute set appears as lib.nixos in the flake, or can be imported
@ -30,4 +32,10 @@ in
inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal) inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal)
evalModules evalModules
; ;
inherit (testing-lib)
evalTest
runTest
;
} }

31
nixos/lib/testing-python.nix
View file

@ -12,6 +12,10 @@
with pkgs; with pkgs;
let
nixos-lib = import ./default.nix { inherit (pkgs) lib; };
in
rec { rec {
inherit pkgs; inherit pkgs;
@ -166,26 +170,15 @@ rec {
${lib.optionalString (interactive) "--add-flags --interactive"} ${lib.optionalString (interactive) "--add-flags --interactive"}
''); '');
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; };
runTest = module: (evalTest module).config.run; runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; };
testModules = [ extraTestModule = {
./testing/driver.nix config = {
./testing/interactive.nix hostPkgs = pkgs;
./testing/legacy.nix minimalResult = hydra;
./testing/meta.nix };
./testing/name.nix };
./testing/network.nix
./testing/nodes.nix
./testing/pkgs.nix
./testing/run.nix
./testing/testScript.nix
{
config = {
hostPkgs = pkgs;
};
}
];
# Make a full-blown test # Make a full-blown test
makeTest = makeTest =

24
nixos/lib/testing/default.nix Normal file
View file

@ -0,0 +1,24 @@
{ lib }:
let
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
runTest = module: (evalTest module).config.result;
testModules = [
./call-test.nix
./driver.nix
./interactive.nix
./legacy.nix
./meta.nix
./name.nix
./network.nix
./nodes.nix
./pkgs.nix
./run.nix
./testScript.nix
];
in
{
inherit evalTest runTest testModules;
}