From 2f2c2161a3c52628412bd2d74b29a1f62b01bd70 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sat, 3 Dec 2022 22:37:38 -0800 Subject: [PATCH] parametrize services on the domain name --- haproxy/config.nix | 87 +------ haproxy/configcreator.nix | 464 ++++++++++++++++++++++++++++++-------- haproxy/mkconfig.nix | 18 +- haproxy/siteconfig.nix | 88 -------- ttrss/config.nix | 3 +- ttrss/mkconfig.nix | 2 + 6 files changed, 377 insertions(+), 285 deletions(-) delete mode 100644 haproxy/siteconfig.nix diff --git a/haproxy/config.nix b/haproxy/config.nix index 79a2497..88bc688 100644 --- a/haproxy/config.nix +++ b/haproxy/config.nix @@ -5,96 +5,21 @@ }: { configDir ? "/etc/haproxy" , configFile ? "haproxy.cfg" -, frontends ? [] -, backends ? [] -, certPath -, user ? "haproxy" -, group ? "haproxy" - -, statsEnable ? false -, statsPort ? 8404 -, statsUri ? "/stats" -, statsRefresh ? "10s" -, prometheusStatsUri ? null +, config }: with builtins; +with lib.attrsets; +with lib.lists; with lib.strings; let - stats = if statsEnable then "" else '' - frontend stats - bind localhost:${toString statsPort} - mode http - stats enable - # stats hide-version - stats uri ${statsUri} - stats refresh ${statsRefresh} - '' + (if prometheusStatsUri == null then "" else '' - http-request use-service prometheus-exporter if { path ${prometheusStatsUri} } - ''); - - indent = spaces: content: - concatMapStrings - (x: spaces + x + "\n") - (splitString "\n" content); - - frontends_str = concatMapStrings (acl: indent " " acl) frontends; - backends_str = concatStringsSep "\n" backends; + configcreator = pkgs.callPackage ./configcreator.nix {}; in utils.mkConfigFile { name = configFile; dir = configDir; - content = '' - global - # Load the plugin handling Let's Encrypt request - # lua-load /etc/haproxy/plugins/haproxy-acme-validation-plugin-0.1.1/acme-http01-webroot.lua - - # Silence a warning issued by haproxy. Using 2048 - # instead of the default 1024 makes the connection stronger. - tune.ssl.default-dh-param 2048 - - maxconn 20000 - - user ${user} - group ${group} - - log /dev/log local0 info - - # Include ssl cipher in log output. - # tune.ssl.capture-cipherlist-size 800 - - defaults - log global - option httplog - - timeout connect 10s - timeout client 15s - timeout server 30s - timeout queue 100s - - frontend http-to-https - mode http - bind *:80 - redirect scheme https code 301 if !{ ssl_fc } - - ${stats} - - frontend https - mode http - - # log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %sslv %sslc %[ssl_fc_cipherlist_str]" - - bind *:443 ssl crt ${certPath} - http-request set-header X-Forwarded-Port %[dst_port] - http-request set-header X-Forwarded-For %[src] - http-request add-header X-Forwarded-Proto https - http-response set-header Strict-Transport-Security "max-age=15552000; includeSubDomains; preload;" - - ${frontends_str} - - ${backends_str} - ''; -} + content = configcreator.render (configcreator.default config); +} diff --git a/haproxy/configcreator.nix b/haproxy/configcreator.nix index 1f64763..9fa5c13 100644 --- a/haproxy/configcreator.nix +++ b/haproxy/configcreator.nix @@ -2,18 +2,250 @@ }: with builtins; +with lib; with lib.attrsets; with lib.lists; with lib.strings; -rec { +let + augmentedContent = fieldName: rules: parent: set: + let + print = {rule = k: parent: v: + assert assertMsg (isString v || isInt v) "cannot print key '${fieldName}.${k}' of type '${typeOf v}', should be string or int instead"; + "${k} ${toString v}";}; + + matchingRule = k: v: findFirst (rule: rule.match k parent v) print rules; + + augment = parent: k: v: + let + match = matchingRule k v; + rule = if hasAttr "rule" match then match.rule else null; + rules = if hasAttr "rules" match then match.rules else null; + indent = map (x: if hasAttr "indent" match then match.indent + x else x); + headerFn = if hasAttr "header" match then match.header else null; + header = optional (headerFn != null) (headerFn k); + trailer = optional (headerFn != null) ""; + in + if rule != null + then rule k parent v + else + assert assertMsg (isAttrs v) "attempt to apply rules on key '${toString k}' which is a '${typeOf v}' but should be a set:\n${toString v}"; + header ++ indent (augmentedContent "${fieldName}.${k}" rules (parent ++ [k]) v) ++ trailer; + + augmented = mapAttrsToList (augment parent) ( + assert assertMsg (isAttrs set) "attempt to apply rules on field ${fieldName} having type '${typeOf set}':\n${toString set}"; + set + ); + in + flatten augmented; + + updateByPath = path: fn: set: + if hasAttrByPath path set then + recursiveUpdate set (setAttrByPath path (fn (getAttrFromPath path set))) + else + set; + + schema = + let + mkRule = + { redirect ? false + , scheme ? "https" + , code ? null + , condition ? null + }: + concatStringsRecursive " " [ + (optional redirect "redirect") + "scheme" scheme + (optional (code != null) "code ${toString code}") + (optional (condition != null) "if ${condition}") + ]; + + mkBind = + { addr + , ssl ? false + , crt ? null + }: + concatStringsRecursive " " [ + "bind" + addr + (optional ssl "ssl") + (optional (crt != null) "crt ${crt}") + ]; + + mkServer = + { name + , address + , balance ? null + , check ? null + , httpcheck ? null + , forwardfor ? true + }: + [ + "mode http" + (optional forwardfor "option forwardfor") + (optional (httpcheck != null) "option httpchk ${httpcheck}") + (optional (balance != null) "balance ${balance}") + (concatStringsRecursive " " [ + "server" + name + address + (optionals (check != null) (mapAttrsToList (k: v: "${k} ${v}") check)) + ]) + ]; + in [ + { + match = k: parent: v: k == "defaults"; + indent = " "; + header = k: k; + rules = [ + { + match = k: parent: v: k == "timeout"; + rule = k: parent: v: mapAttrsToList (k1: v1: "${k} ${k1} ${v1}") v; + } + ]; + } + { + match = k: parent: v: k == "global"; + indent = " "; + header = k: k; + rules = []; + } + { + match = k: parent: v: k == "frontend"; + rules = [ + { + match = k: parent: v: true; + header = k: "frontend " + k; + indent = " "; + rules = [ + { + match = k: parent: v: k == "rules"; + rule = k: parent: v: map mkRule v; + } + { + match = k: parent: v: k == "bind" && isAttrs v; + rule = k: parent: v: mkBind v; + } + { + match = k: parent: v: k == "use_backend"; + rule = k: parent: v: + let + use = name: value: "use_backend ${name} ${toString value}"; + in + if isList v then + map (v: use v.name v.value) v + else + use v.name v.value; + } + { + match = k: parent: v: true ; + rule = k: parent: v: + let + l = prefix: v: + if isAttrs v then + mapAttrsToList (k: v: l "${prefix} ${k}" v) v + else if isList v then + map (l prefix) v + else if isBool v then + optional v prefix + else + assert assertMsg (isString v) "value for field ${k} should be a string, bool, attr or list, got: ${typeOf v}"; + "${prefix} ${v}"; + in + l k v; + } + ]; + } + ]; + } + { + match = k: parent: v: k == "backend"; + rules = [ + { + match = k: parent: v: true; + header = k: "backend " + k; + indent = " "; + rules = [ + { + match = k: parent: v: k == "options"; + rule = k: parent: v: v; + } + { + match = k: parent: v: k == "servers"; + rule = k: parent: v: map mkServer v; + } + ]; + } + ]; + } + # { + # match = k: v: k == "plugins"; + # rule = k: v: mkPlugins v; + # } + ]; + + + concatStringsRecursive = sep: strings: + concatStringsSep sep (flatten strings); + + recursiveMerge = attrList: + let f = attrPath: + zipAttrsWith (n: values: + if tail values == [] then + head values + else if all isList values then + concatLists values + else if all isAttrs values then + f (attrPath ++ [n]) values + else + last values + ); + in f [] attrList; + + assertHasAttr = name: attrPath: v: + assertMsg + (hasAttrByPath attrPath v) + "no ${last attrPath} defined in config for site ${name}.${concatStringsSep "." (init attrPath)}, found attr names: ${toString (attrNames (getAttrFromPath (init attrPath) v))}"; + + # Takes a function producing a [nameValuePair], applies it to + # all name-value pair in the given set and merges the resulting + # [[nameValuePair]]. + mapAttrsFlatten = f: set: listToAttrs (concatLists (mapAttrsToList f set)); + + mapIfIsAttrs = f: value: + if isAttrs value + then f value + else value; + + flattenAttrs = sep: cond: set: + let + recurse = mapIfIsAttrs (mapAttrsFlatten ( + n: v: let + result = recurse v; + in + if isAttrs result && cond n v + then mapAttrsToList (n2: v2: nameValuePair "${n}${sep}${n2}" v2) result + else [(nameValuePair n result)] + )); + in recurse set; +in +{ + inherit updateByPath recursiveMerge; + default = { user , group , certPath + , plugins ? [] , stats ? null , debug ? false + , sites ? {} }: { + # inherit plugins; + global = { + # Load the plugin handling Let's Encrypt request + # lua-load /etc/haproxy/plugins/haproxy-acme-validation-plugin-0.1.1/acme-http01-webroot.lua + # Silence a warning issued by haproxy. Using 2048 # instead of the default 1024 makes the connection stronger. "tune.ssl.default-dh-param" = 2048; @@ -49,25 +281,77 @@ rec { condition = "!{ ssl_fc }"; } ]; + backend = {}; }; - https = { - mode = "http"; - bind = { - addr = "*:443"; - ssl = true; - crt = certPath; - }; - } // optionalAttrs (debug) { - log-format = "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %sslv %sslc %[ssl_fc_cipherlist_str]"; + https = ( + let + r = ( + [{ + mode = "http"; + bind = { + addr = "*:443"; + ssl = true; + crt = certPath; + }; + + http-request = { + set-header = [ + "X-Forwarded-Port %[dst_port]" + "X-Forwarded-For %[src]" + ]; + add-header = [ + "X-Forwarded-Proto https" + ]; + }; + + http-response = [ + ''set-header Strict-Transport-Security "max-age=15552000; includeSubDomains; preload;"'' + ]; + + # acl = flatten (mapAttrsToList (name: config: + # assert assertHasAttr name ["frontend" "acl"] config; + # config.frontend.acl + # ) sites); + # use_backend = mapAttrsToList (name: config: + # assert assertHasAttr name ["frontend" "use_backend"] config; + # nameValuePair name config.frontend.use_backend + # ) sites; + }] + ++ (mapAttrsToList (name: config: + assert assertHasAttr name ["frontend"] config; + #(filterAttrs (k: v: k != "use_backend" && k != "acl") + # (mapAttrsRecursive + # (ks: v: optionalAttrs (hasAttrByPath ["frontend" "use_backend"] v) (setAttrByPath ["frontend" "use_backend"] (nameValuePair name v.frontend.use_backend))) + # config.frontend + (updateByPath ["frontend" "use_backend"] (x: (nameValuePair name x)) config).frontend + #) + ) sites) + ++ (mapAttrsToList (name: config: + if (hasAttr "debugHeaders" config && (getAttr "debugHeaders" config) != null) then { + option = "httplog"; + http-request = { + capture = "req.hdrs len 512 if ${config.debugHeaders}"; + }; + log-format = ''"%ci:%cp [%tr] %ft [[%hr]] %hs %{+Q}r"''; + } else {} + ) sites) + ); + in + recursiveMerge r + ) + // optionalAttrs (debug) { + log-format = ''"%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %sslv %sslc %[ssl_fc_cipherlist_str]"''; }; } // optionalAttrs (stats != null) (let stats_ = { + enable = true; port = 8404; uri = "/stats"; refresh = "10s"; prometheusUri = null; + hide-version = false; } // stats; in { @@ -75,107 +359,87 @@ rec { bind = "localhost:${toString stats_.port}"; mode = "http"; stats = { - enable = true; - hide-version = false; + enable = stats_.enable; + hide-version = stats_.hide-version; uri = stats_.uri; refresh = stats_.refresh; - } // optionalAttrs (stats_.prometheusUri != null) { - http-request = [ - "use-service prometheus-exporter if { path ${stats_.prometheusUri} }" - ]; }; + } // optionalAttrs (stats_.prometheusUri != null) { + http-request = [ + "use-service prometheus-exporter if { path ${stats_.prometheusUri} }" + ]; }; }); + + backend = + mapAttrs' (name: config: + assert assertMsg (hasAttr "backend" config) "no backend defined in config for site ${name}, found attr names: ${toString (attrNames config)}"; + nameValuePair name config.backend) + sites; + # inherit backend; + # backend = + # let + # b = backend: nameValuePair backend.name {inherit (backend) options;}; + # in + # listToAttrs (flatten (map (s: mapAttrsToList b s.backends) sites)); }; + + # mapIfHasAttr = f: attr: set: + # if hasAttr attr set + # then f (getAttr attr set) + # else set; - mkRule = - { redirect ? false - , scheme ? "https" - , code ? null - , condition ? null - }: - concatStringsSep " " (flatten [ - (optional redirect "redirect") - scheme - (optional (code != null) "code ${toString code}") - (optional (condition != null) "if ${condition}") - ]); + # Lua's import system requires the import path to be something like: + # + # /nix/store/123-name// + # + # Then the lua-prepend-path can be: + # + # /nix/store/123-name/?/ + # + # Then when lua code imports , it will search in the + # prepend paths and replace the question mark with the + # name to get a match. + # + # But the config.source is actually without the name: + # + # /nix/store/123-name/ + # + # This requires us to create a new directory structure and we're + # using a linkFarm for this. + # pluginLinks = configs: + # let + # mkLink = config: { + # inherit (config) name; + # path = config.source; + # }; - mkBind = - { addr - , ssl ? false - , crt ? null - }: - concatStringsSep " " (flatten [ - addr - (optional ssl "ssl") - (optional (crt != null) "crt ${crt}") - ]); + # links = pkgs.linkFarm "haproxyplugins" (map mkLink configs); + # in + # map (config: + # "lua-prepend-path ${links}/?/${config.init}" + # ) configs; - augmentedContent = fieldName: rules: set: - let - print = {rule = k: v: - assert lib.assertMsg (isString v || isInt v) "cannot print key '${k}' of type '${typeOf v}', should be string or int instead"; - "${k} ${toString v}";}; + # loadPlugins = links: configs: + # let + # mustLoad = config: hasAttr "load" config && config.load; + # in + # concatMap + # (config: optional (mustLoad config) "lua-load ${links}/${config.name}/${config.init}") + # configs; - matchingRule = k: v: findFirst (rule: rule.match k v) print rules; + # mkPlugins = configs: + # { name + # , init + # , source + # , load ? false + # }: + # let + # path = "lua-prepend-path ${links}/?/${init}" - augment = k: v: - let - match = matchingRule k v; - rule = if hasAttr "rule" match then match.rule else null; - rules = if hasAttr "rules" match then match.rules else null; - in - if rule != null - then rule k v - else - assert lib.assertMsg (isAttrs v) "attempt to apply rules on key '${k}' which is a '${typeOf v}' but should be a set"; - augmentedContent k rules v; - in - flatten (mapAttrsToList augment ( - assert lib.assertMsg (isAttrs set) "attempt to apply rules on field ${fieldName} having type '${typeOf set}'"; - set - )); - - # mkSection = name: config: - schema = [ - { - match = k: v: k == "defaults"; - rules = [ - { - match = k: v: k == "timeout"; - rule = k: v: mapAttrsToList (k1: v1: "${k}.${k1} ${v1}") v; - } - ]; - } - { - match = k: v: k == "global"; - rules = []; - } - { - match = k: v: k == "frontend"; - rules = [ - { - match = k: v: true; - rules = [ - { - match = k: v: k == "rules"; - rule = k: v: map mkRule v; - } - { - match = k: v: k == "bind" && isAttrs v; - rule = k: v: mkBind v; - } - { - match = k: v: k == "http-request" || k == "http-response"; - rule = k: v: v; - } - ]; - } - ]; - } - ]; + # concatStringsSep " " (flatten [ + # ]); render = config: - concatStringsSep "\n" (augmentedContent name schema config); + concatStringsSep "\n" (augmentedContent "" schema [] config); } diff --git a/haproxy/mkconfig.nix b/haproxy/mkconfig.nix index 237fff6..11cfaf5 100644 --- a/haproxy/mkconfig.nix +++ b/haproxy/mkconfig.nix @@ -3,27 +3,15 @@ { name , configDir , configFile -, user -, group -, statsEnable ? false -, statsPort ? null -, prometheusStatsUri ? null -, certPath ? null -, frontends ? [] -, backends ? [] +, config , dependsOn ? {} }: { inherit name configDir configFile; - inherit user group; + inherit (config) user group; pkg = HaproxyConfig { inherit configDir configFile; - inherit user group; - inherit statsEnable statsPort; - inherit prometheusStatsUri; - inherit certPath; - - inherit frontends backends; + inherit config; }; inherit dependsOn; diff --git a/haproxy/siteconfig.nix b/haproxy/siteconfig.nix deleted file mode 100644 index c3c7dc5..0000000 --- a/haproxy/siteconfig.nix +++ /dev/null @@ -1,88 +0,0 @@ -{ stdenv -, pkgs -, lib -}: -{ serviceName -, servers ? [] -, httpcheck ? null -, balance ? null -, phpFastcgi ? false -, phpDocroot ? null -, phpIndex ? "index.php" -, extraUseBackendConditions ? {} -, extraFrontendOptions ? [] -, extraBackendOptions ? [] - -, debugHeaders ? false -}: - -with lib; -with lib.lists; -with lib.attrsets; -let - indent = map (x: " " + x); - - mkServer = i: s: - let - proto = optional phpFastcgi "proto fcgi"; - in - concatStringsSep " " ( - [ - "server ${serviceName}${toString i} ${s.address}" - ] - ++ proto - ++ (optional (hasAttr "check" s && s.check != null) ( - concatStrings (["check"] ++ (map (k: if !hasAttr k s.check then "" else " ${k} ${getAttr k s.check}") ["inter" "downinter" "fall" "rise"])) - )) - ); - - serverslines = imap1 mkServer servers; - - backend = - ( - concatStringsSep "\n" ( - [ - "backend ${serviceName}" - ] - ++ indent ( - [ - "mode http" - "option forwardfor" - ] - ++ extraBackendOptions - ++ optional (balance != null) "balance ${balance}" - ++ optional (httpcheck != null) "option httpchk ${httpcheck}" - ++ optional phpFastcgi "use-fcgi-app ${serviceName}-php-fpm" - ++ serverslines - ) - ++ [""]) # final newline - ) + - (if !phpFastcgi then "" else '' - - fcgi-app ${serviceName}-php-fpm - log-stderr global - docroot ${phpDocroot} - index ${phpIndex} - path-info ^(/.+\.php)(/.*)?$ - ''); - - extraAclsCondition = concatStrings (mapAttrsToList (k: v: "\nacl acl_${serviceName}_${k} ${v}") extraUseBackendConditions); - - extraAclsOr = concatStrings (mapAttrsToList (k: v: " OR acl_${serviceName}_${k}") extraUseBackendConditions); -in -{ - frontend = '' - acl acl_${serviceName} hdr_beg(host) ${serviceName}.${extraAclsCondition} - '' - + concatMapStrings (x: x + "\n") extraFrontendOptions - + concatMapStrings (x: x + "\n") (optionals debugHeaders [ - "option httplog" - "http-request capture req.hdrs len 512 if acl_${serviceName}${extraAclsOr}" - ''log-format "%ci:%cp [%tr] %ft [[%hr]] %hs %{+Q}r"'' - ]) - + '' - use_backend ${serviceName} if acl_${serviceName}${extraAclsOr} - ''; - - inherit backend; -} diff --git a/ttrss/config.nix b/ttrss/config.nix index e15d5b4..dd1411c 100644 --- a/ttrss/config.nix +++ b/ttrss/config.nix @@ -6,6 +6,7 @@ , name ? "ttrss" , user ? "http" , group ? "http" +, domain , lock_directory , cache_directory , feed_icons_directory @@ -86,7 +87,7 @@ stdenv.mkDerivation rec { buildCommand = let - configFile = pkgs.writeText "config.php" (asTtrssConfig (config "https://${name}.tiserbox.com/")); + configFile = pkgs.writeText "config.php" (asTtrssConfig (config "https://${name}.${domain}/")); dr = dirOf document_root; in '' diff --git a/ttrss/mkconfig.nix b/ttrss/mkconfig.nix index f224126..3bf4111 100644 --- a/ttrss/mkconfig.nix +++ b/ttrss/mkconfig.nix @@ -3,6 +3,7 @@ { name , user , group +, domain , serviceName , document_root , lock_directory @@ -26,6 +27,7 @@ name = serviceName; inherit document_root lock_directory cache_directory feed_icons_directory; inherit user group; + inherit domain; inherit db_host db_port db_username db_password db_database; inherit enabled_plugins;