From 66398fe159368dc245bca4f274562ed163959f37 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sun, 3 Dec 2023 23:56:25 -0800 Subject: [PATCH 1/4] use nix-fast-builds in CI --- .github/workflows/test.yml | 8 +++----- .gitignore | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39c57a2..9a2ade6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,8 +17,6 @@ jobs: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: | - nix flake \ - --print-build-logs \ - --option keep-going true \ - --show-trace \ - check + nix run github:Mic92/nix-fast-build -- \ + --skip-cached --no-nom \ + --flake ".#checks.$(nix eval --raw --impure --expr builtins.currentSystem)" diff --git a/.gitignore b/.gitignore index 0214a90..5ce57ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.qcow2 -result \ No newline at end of file +result +result-* \ No newline at end of file From 5f4ea7acfb376264debf7dcfe326a6fd621657c9 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sun, 3 Dec 2023 23:33:39 -0800 Subject: [PATCH 2/4] wait for open port in all tests to reduce flakiness --- test/vm/ldap.nix | 1 + test/vm/monitoring.nix | 1 + test/vm/postgresql.nix | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/test/vm/ldap.nix b/test/vm/ldap.nix index 6b8ee60..967757a 100644 --- a/test/vm/ldap.nix +++ b/test/vm/ldap.nix @@ -34,6 +34,7 @@ start_all() server.wait_for_unit("lldap.service") + server.wait_for_open_port(${toString nodes.server.services.lldap.settings.http_port}) with subtest("fail without authenticating"): client.fail( diff --git a/test/vm/monitoring.nix b/test/vm/monitoring.nix index 12723ce..e9ee204 100644 --- a/test/vm/monitoring.nix +++ b/test/vm/monitoring.nix @@ -28,6 +28,7 @@ testScript = { nodes, ... }: '' start_all() machine.wait_for_unit("grafana.service") + machine.wait_for_open_port(${toString nodes.machine.services.grafana.settings.server.http_port}) def curl_req(password, wantStatus, endpoint): response = machine.wait_until_succeeds("curl -i http://admin:{password}@localhost:3000{endpoint}".format(password=password, endpoint=endpoint), timeout=10) diff --git a/test/vm/postgresql.nix b/test/vm/postgresql.nix index c8ef789..61872c3 100644 --- a/test/vm/postgresql.nix +++ b/test/vm/postgresql.nix @@ -19,6 +19,7 @@ testScript = { nodes, ... }: '' start_all() machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) def peer_cmd(user, database): return "sudo -u me psql -U {user} {db} --command \"\"".format(user=user, db=database) @@ -60,6 +61,7 @@ testScript = { nodes, ... }: '' start_all() machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) def peer_cmd(user, database): return "sudo -u me psql -U {user} {db} --command \"\"".format(user=user, db=database) @@ -101,6 +103,7 @@ testScript = { nodes, ... }: '' start_all() machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) def peer_cmd(user, database): return "sudo -u me psql -U {user} {db} --command \"\"".format(user=user, db=database) @@ -147,6 +150,7 @@ testScript = { nodes, ... }: '' start_all() machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) def peer_cmd(user, database): return "sudo -u me psql -U {user} {db} --command \"\"".format(user=user, db=database) From 13a90f1ad402c40185ebc03cfd82e7e95f8f629f Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sun, 3 Dec 2023 21:42:09 -0800 Subject: [PATCH 3/4] avoid testing nix store path hash which makes tests brittle --- test/modules/arr.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/modules/arr.nix b/test/modules/arr.nix index 9c0bd57..60a3677 100644 --- a/test/modules/arr.nix +++ b/test/modules/arr.nix @@ -28,8 +28,11 @@ let m ]; }).config; + + systemdRedacted = lib.filterAttrsRecursive (n: v: n != "preStart") cfg.systemd; in { - inherit (cfg) systemd services users; + inherit (cfg) services users; + systemd = systemdRedacted; shb = { inherit (cfg.shb) backup nginx; }; }; in @@ -54,7 +57,6 @@ in testRadarr = { expected = { systemd.services.radarr = { - preStart = "ln -fs /nix/store/z7gk6xfj51sr1n1bjj6lsadjrwjxzc5d--config.xml /var/lib/radarr/config.xml.template\nrm /var/lib/radarr/config.xml || :\nsed -e \"s|%APIKEY%|$(cat /run/radarr/apikey)|\" /var/lib/radarr/config.xml.template > /var/lib/radarr/config.xml\n"; serviceConfig = { StateDirectoryMode = "0750"; UMask = "0027"; @@ -122,7 +124,6 @@ in testRadarrWithBackup = { expected = { systemd.services.radarr = { - preStart = "ln -fs /nix/store/z7gk6xfj51sr1n1bjj6lsadjrwjxzc5d--config.xml /var/lib/radarr/config.xml.template\nrm /var/lib/radarr/config.xml || :\nsed -e \"s|%APIKEY%|$(cat /run/radarr/apikey)|\" /var/lib/radarr/config.xml.template > /var/lib/radarr/config.xml\n"; serviceConfig = { StateDirectoryMode = "0750"; UMask = "0027"; From 0242ae26c480e170e9d736f08f9b52258faf643f Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sun, 3 Dec 2023 23:33:55 -0800 Subject: [PATCH 4/4] fix ensure clauses in postgresql fixes #35 --- flake.lock | 32 ++++++++++++++++---------------- modules/blocks/postgresql.nix | 8 ++------ test/modules/postgresql.nix | 32 ++++++++------------------------ test/vm/postgresql.nix | 28 ++++++++++++++-------------- 4 files changed, 40 insertions(+), 60 deletions(-) diff --git a/flake.lock b/flake.lock index e954d01..56021d4 100644 --- a/flake.lock +++ b/flake.lock @@ -35,11 +35,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1687412861, - "narHash": "sha256-Z/g0wbL68C+mSGerYS2quv9FXQ1RRP082cAC0Bh4vcs=", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", "owner": "nixos", "repo": "nixpkgs", - "rev": "e603dc5f061ca1d8a19b3ede6a8cf9c9fcba6cdc", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", "type": "github" }, "original": { @@ -51,11 +51,11 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1687031877, - "narHash": "sha256-yMFcVeI+kZ6KD2QBrFPNsvBrLq2Gt//D0baHByMrjFY=", + "lastModified": 1701568804, + "narHash": "sha256-iwr1fjOCvlirVL/xNvOTwY9kg3L/F3TC/7yh/QszaPI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e2e2059d19668dab1744301b8b0e821e3aae9c99", + "rev": "dc01248a9c946953ad4d438b0a626f5c987a93e4", "type": "github" }, "original": { @@ -67,11 +67,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1686979235, - "narHash": "sha256-gBlBtk+KrezFkfMrZw6uwTuA7YWtbFciiS14mEoTCo0=", + "lastModified": 1701336116, + "narHash": "sha256-kEmpezCR/FpITc6yMbAh4WrOCiT2zg5pSjnKrq51h5Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7cc30fd5372ddafb3373c318507d9932bd74aafe", + "rev": "f5c27c6136db4d76c30e533c20517df6864c46ee", "type": "github" }, "original": { @@ -84,11 +84,11 @@ "nmdsrc": { "flake": false, "locked": { - "lastModified": 1687627428, - "narHash": "sha256-7zGfXuNS5RHqhpEdz2fwrtqvF86JRo5U1hrxZSYgcm8=", + "lastModified": 1701431551, + "narHash": "sha256-5HPHG1u3koaWHG/TXHl5/YxYPYOuKc58104btrD8ypE=", "ref": "refs/heads/master", - "rev": "824a380546b5d0d0eb701ff8cd5dbafb360750ff", - "revCount": 63, + "rev": "f18defadcc25e69e95b04493ee02682005472255", + "revCount": 65, "type": "git", "url": "https://git.sr.ht/~rycee/nmd" }, @@ -112,11 +112,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1687398569, - "narHash": "sha256-e/umuIKFcFtZtWeX369Hbdt9r+GQ48moDmlTcyHWL28=", + "lastModified": 1701572436, + "narHash": "sha256-0anfOQqDend6kSuF8CmOSAZsiAS1nwOsin5VQukh6Q4=", "owner": "Mic92", "repo": "sops-nix", - "rev": "2ff6973350682f8d16371f8c071a304b8067f192", + "rev": "8bca48cb9a12bbd8766f359ad00336924e91b7f7", "type": "github" }, "original": { diff --git a/modules/blocks/postgresql.nix b/modules/blocks/postgresql.nix index 5ddd813..0139404 100644 --- a/modules/blocks/postgresql.nix +++ b/modules/blocks/postgresql.nix @@ -72,12 +72,8 @@ in services.postgresql.ensureDatabases = map ({ database, ... }: database) ensureCfgs; services.postgresql.ensureUsers = map ({ username, database, ... }: { name = username; - ensurePermissions = { - "DATABASE ${database}" = "ALL PRIVILEGES"; - }; - ensureClauses = { - "login" = true; - }; + ensureDBOwnership = true; + ensureClauses.login = true; }) ensureCfgs; }; diff --git a/test/modules/postgresql.nix b/test/modules/postgresql.nix index 91db948..863fa75 100644 --- a/test/modules/postgresql.nix +++ b/test/modules/postgresql.nix @@ -64,9 +64,7 @@ in enable = true; ensureUsers = [{ name = "myuser"; - ensurePermissions = { - "DATABASE mydatabase" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; @@ -92,9 +90,7 @@ in enable = true; ensureUsers = [{ name = "myuser"; - ensurePermissions = { - "DATABASE mydatabase" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; @@ -131,18 +127,14 @@ in ensureUsers = [ { name = "user1"; - ensurePermissions = { - "DATABASE db1" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; } { name = "user2"; - ensurePermissions = { - "DATABASE db2" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; @@ -174,18 +166,14 @@ in ensureUsers = [ { name = "user1"; - ensurePermissions = { - "DATABASE db1" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; } { name = "user2"; - ensurePermissions = { - "DATABASE db2" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; @@ -230,18 +218,14 @@ in ensureUsers = [ { name = "user1"; - ensurePermissions = { - "DATABASE db1" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; } { name = "user2"; - ensurePermissions = { - "DATABASE db2" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; ensureClauses = { "login" = true; }; diff --git a/test/vm/postgresql.nix b/test/vm/postgresql.nix index 61872c3..28db06c 100644 --- a/test/vm/postgresql.nix +++ b/test/vm/postgresql.nix @@ -11,7 +11,7 @@ shb.postgresql.ensures = [ { username = "me"; - database = "mine"; + database = "me"; } ]; }; @@ -25,10 +25,10 @@ return "sudo -u me psql -U {user} {db} --command \"\"".format(user=user, db=database) with subtest("cannot login because of missing user"): - machine.fail(peer_cmd("me", "mine"), timeout=10) + machine.fail(peer_cmd("me", "me"), timeout=10) with subtest("cannot login with unknown user"): - machine.fail(peer_cmd("notme", "mine"), timeout=10) + machine.fail(peer_cmd("notme", "me"), timeout=10) with subtest("cannot login to unknown database"): machine.fail(peer_cmd("me", "notmine"), timeout=10) @@ -53,7 +53,7 @@ shb.postgresql.ensures = [ { username = "me"; - database = "mine"; + database = "me"; } ]; }; @@ -70,16 +70,16 @@ return "psql -h 127.0.0.1 -p {port} -U {user} {db} --command \"\"".format(user=user, db=database, port=port) with subtest("can login with provisioned user and database"): - machine.succeed(peer_cmd("me", "mine"), timeout=10) + machine.succeed(peer_cmd("me", "me"), timeout=10) with subtest("cannot login with unknown user"): - machine.fail(peer_cmd("notme", "mine"), timeout=10) + machine.fail(peer_cmd("notme", "me"), timeout=10) with subtest("cannot login to unknown database"): machine.fail(peer_cmd("me", "notmine"), timeout=10) with subtest("cannot login with tcpip"): - machine.fail(tcpip_cmd("me", "mine", "5432"), timeout=10) + machine.fail(tcpip_cmd("me", "me", "5432"), timeout=10) ''; }; @@ -95,7 +95,7 @@ shb.postgresql.ensures = [ { username = "me"; - database = "mine"; + database = "me"; } ]; }; @@ -112,10 +112,10 @@ return "psql -h 127.0.0.1 -p {port} -U {user} {db} --command \"\"".format(user=user, db=database, port=port) with subtest("cannot login without existing user"): - machine.fail(peer_cmd("me", "mine"), timeout=10) + machine.fail(peer_cmd("me", "me"), timeout=10) with subtest("cannot login with user without password"): - machine.fail(tcpip_cmd("me", "mine", "5432"), timeout=10) + machine.fail(tcpip_cmd("me", "me", "5432"), timeout=10) ''; }; @@ -141,7 +141,7 @@ shb.postgresql.ensures = [ { username = "me"; - database = "mine"; + database = "me"; passwordFile = "/run/dbsecret"; } ]; @@ -159,13 +159,13 @@ return "PGPASSWORD={password} psql -h 127.0.0.1 -p {port} -U {user} {db} --command \"\"".format(user=user, db=database, port=port, password=password) with subtest("can peer login with provisioned user and database"): - machine.succeed(peer_cmd("me", "mine"), timeout=10) + machine.succeed(peer_cmd("me", "me"), timeout=10) with subtest("can tcpip login with provisioned user and database"): - machine.succeed(tcpip_cmd("me", "mine", "5432", "secretpw"), timeout=10) + machine.succeed(tcpip_cmd("me", "me", "5432", "secretpw"), timeout=10) with subtest("cannot tcpip login with wrong password"): - machine.fail(tcpip_cmd("me", "mine", "5432", "oops"), timeout=10) + machine.fail(tcpip_cmd("me", "me", "5432", "oops"), timeout=10) ''; }; }