From 5cb7263c176e0527822dd9ce252c876deb083961 Mon Sep 17 00:00:00 2001 From: Viv Lim Date: Sat, 4 Mar 2023 18:42:59 -0800 Subject: [PATCH] (messily) refactor container definition so it's reusable --- configs/backend.nix | 285 +++++++++------------------------ modules/mastodon_container.nix | 189 ++++++++++++++++++++++ 2 files changed, 268 insertions(+), 206 deletions(-) create mode 100644 modules/mastodon_container.nix diff --git a/configs/backend.nix b/configs/backend.nix index b91061c..7987ec4 100644 --- a/configs/backend.nix +++ b/configs/backend.nix @@ -1,11 +1,74 @@ { config, pkgs, nestedContainerExtras, ... }: +let + mastodonContainerFactory = import ../modules/mastodon_container.nix { }; + prodConfig = { + containerName = "mastodon"; + hostAddress = "192.168.42.11"; + domain = "snoot.tube"; + localAddress = "192.168.42.12"; + useElasticsearch = true; + mastodonPackage = pkgs.mastodonFork; + forwardPorts = [ + { + containerPort = 443; + hostPort = 443; + protocol = "tcp"; + } + { + containerPort = 80; + hostPort = 80; + protocol = "tcp"; + } + ]; + imports = nestedContainerExtras.imports; + disabledModules = nestedContainerExtras.disabledModules; + acme = { + acceptTerms = true; + defaults.email = "vivlim@pm.me"; + defaults.server = + "https://localhost"; # try to use localhost as acme server. this request *will* fail, and we'll fall back to self-signed + #defaults.enableDebugLogs = true; + }; + smtp = { + createLocally = false; + authenticate = false; + fromAddress = "snoot-tube-mastodon@vvn.space"; + host = + "192.168.1.7"; # ts ip isn't reachable from the container. maybe need a mail relay on the host? use lan ip for now "100.109.126.18"; # sky-reflected-in-mirrors ts ip + port = 25; + }; + mastodonExtraConfig = { + ALTERNATE_DOMAINS = "dev.snoot.tube,awake.snoot.tube"; + #EMAIL_DOMAIN_ALLOWLIST = "vvn.space"; + SMTP_AUTH_METHOD = "none"; + SMTP_OPENSSL_VERIFY_MODE = "none"; + MAX_TOOT_CHARS = "42069"; + AUTHORIZED_FETCH = "true"; + USER_ACTIVE_DAYS = "14"; + STATSD_ADDR = "outer:9125"; + MAX_REACTIONS = "8"; + SEARCH_SCOPE = "public"; + }; + oauth2ProxyUsers = '' + viv + itsonlythee + lifning + Skirmisher + ''; + oauth2ProxyKeys = config.sops.secrets.oauth2_proxy_keys.path; -{ + }; + + prodContainer = (mastodonContainerFactory prodConfig); +in { config.networking.hostName = "mastodon-snoottube"; config.networking.firewall.enable = true; - config.networking.firewall.allowedTCPPorts = [ 80 443 22 19999 19998 19980 19981 9102 4040 ]; - config.networking.firewall.interfaces."ve-+".allowedTCPPorts = [ 3128 ]; # ssh forwarded http proxy - config.networking.firewall.interfaces."ve-+".allowedUDPPorts = [ 9125 ]; # statsd + config.networking.firewall.allowedTCPPorts = + [ 80 443 22 19999 19998 19980 19981 9102 4040 ]; + config.networking.firewall.interfaces."ve-+".allowedTCPPorts = + [ 3128 ]; # ssh forwarded http proxy + config.networking.firewall.interfaces."ve-+".allowedUDPPorts = + [ 9125 ]; # statsd config.networking.firewall.checkReversePath = "loose"; config.networking.extraHosts = '' @@ -17,7 +80,7 @@ # enable containers to reach network config.networking.nat.enable = true; - config.networking.nat.internalInterfaces = ["ve-+"]; + config.networking.nat.internalInterfaces = [ "ve-+" ]; config.networking.nat.externalInterface = "ens18"; config.services.prometheus_exporters = { @@ -33,205 +96,15 @@ user = "root"; monitoringPort = 0; extraArguments = - "-o ExitOnForwardFailure=yes -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -N -R 4433:192.168.42.12:443 -L 3128:localhost:3128 -L 19981:localhost:19980 -L 19998:localhost:19999 -L 4040:localhost:4040 -g -i /root/.ssh/id_ed25519_to_ingress snoot.tube -p 6922"; + "-o ExitOnForwardFailure=yes -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -N -R 4433:${prodConfig.localAddress}:443 -L 3128:localhost:3128 -L 19981:localhost:19980 -L 19998:localhost:19999 -L 4040:localhost:4040 -g -i /root/.ssh/id_ed25519_to_ingress snoot.tube -p 6922"; }]; }; - config.containers = { - mastodon = { - #ephemeral = true; - autoStart = true; - privateNetwork = true; - hostAddress = "192.168.42.11"; - localAddress = "192.168.42.12"; - forwardPorts = [ - { containerPort = 443; hostPort = 443; protocol = "tcp"; } - { containerPort = 80; hostPort = 80; protocol = "tcp"; } - ]; - bindMounts = { - "/var/lib/mastodon" = { hostPath = "/var/lib/mastodon-container/mastodon"; isReadOnly = false;}; - "/var/lib/redis-mastodon" = { hostPath = "/var/lib/mastodon-container/redis-mastodon"; isReadOnly = false;}; - "/var/lib/postgresql" = { hostPath = "/var/lib/mastodon-container/postgresql"; isReadOnly = false;}; - "/var/lib/elasticsearch" = { hostPath = "/var/lib/mastodon-container/elasticsearch"; isReadOnly = false;}; - "/var/lib/acme" = { hostPath = "/var/lib/mastodon-container/acme"; isReadOnly = false;}; - "/var/lib/certs" = { hostPath = "/var/lib/mastodon-container/certs"; isReadOnly = false;}; - "/var/lib/secrets" = { hostPath = "/var/lib/mastodon-container/secrets"; isReadOnly = true;}; - "/var/backup" = { hostPath = "/var/lib/mastodon-container/backup"; isReadOnly = false;}; - }; - config = { pkgs, ... }: { - imports = nestedContainerExtras.imports; - disabledModules = nestedContainerExtras.disabledModules; + config.containers = { mastodon = prodContainer.containerConfig; }; - networking = { - firewall.enable = true; - firewall.allowedTCPPorts = [ 443 80 ]; - proxy.default = "http://outer:3128"; - proxy.noProxy = "127.0.0.1,localhost,outer,192.168.42.11"; - extraHosts = '' - 192.168.42.11 outer - ''; - }; - security = { - acme = { - acceptTerms = true; - defaults.email = "vivlim@pm.me"; - defaults.server = "https://localhost"; # try to use localhost as acme server. this request *will* fail, and we'll fall back to self-signed - #defaults.enableDebugLogs = true; - }; - }; - - services.redis.servers.mastodon = { - enable = true; - bind = "127.0.0.1"; - port = 31637; - }; - - services.mastodon = { - enable = true; - - package = pkgs.mastodonFork; - configureNginx = true; - localDomain = "snoot.tube"; - enableUnixSocket = true; - redis = { - createLocally = true; - host = "127.0.0.1"; - port = 31637; - }; - database = { - createLocally = true; - host = "/run/postgresql"; - port = 5432; - }; - smtp = { - createLocally = false; - authenticate=false; - fromAddress = "snoot-tube-mastodon@vvn.space"; - host = "192.168.1.7"; # ts ip isn't reachable from the container. maybe need a mail relay on the host? use lan ip for now "100.109.126.18"; # sky-reflected-in-mirrors ts ip - port = 25; - }; - extraConfig = { - ALTERNATE_DOMAINS="dev.snoot.tube,awake.snoot.tube"; - #EMAIL_DOMAIN_ALLOWLIST = "vvn.space"; - SMTP_AUTH_METHOD="none"; - SMTP_OPENSSL_VERIFY_MODE="none"; - MAX_TOOT_CHARS="42069"; - AUTHORIZED_FETCH="true"; - USER_ACTIVE_DAYS="14"; - STATSD_ADDR=outer:9125; - MAX_REACTIONS="8"; - SEARCH_SCOPE="public"; - }; - elasticsearch.host = "127.0.0.1"; - #trustedProxy = "20.230.229.78"; # snoot.tube - trustedProxy = "192.168.42.11"; - }; - - # enable pghero - services.postgresql.settings.shared_preload_libraries = "pg_stat_statements"; - services.postgresql.settings."pg_stat_statements.track" = "all"; - - services.postgresqlBackup = { - enable = true; - databases = [ "mastodon" ]; - startAt = "*-*-* 04:15:00"; - location = "/var/backup/postgresql"; - }; - - - nixpkgs.config.allowUnfree = true; # elasticsearch - services.elasticsearch = { - enable = true; - package = pkgs.elasticsearch7; - extraConf = '' - ingest.geoip.downloader.enabled: false - xpack.security.enabled: false - ''; - }; - - - services.oauth2_proxy_mastodon = { - enable = true; - provider = "mastodon"; - keyFile = "/var/lib/secrets/oauth2_proxy_keys"; - setXauthrequest = true; - mastodon = { - mastodon-url = "https://snoot.tube"; - }; - extraConfig = { - }; - email.addresses = '' - viv - itsonlythee - lifning - Skirmisher - ''; - }; - - services.nginx = { - virtualHosts."snoot.tube" = { - locations."/oauth2/" = { - proxyPass = "http://127.0.0.1:4180"; # can't use config.services.oauth2_proxy_mastodon.httpAddress fsr. probably because of weird container module stuff.; - extraConfig = '' - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri; - ''; - }; - locations."/oauth2/auth" = { - proxyPass = "http://127.0.0.1:4180"; # can't use config.services.oauth2_proxy_mastodon.httpAddress fsr. probably because of weird container module stuff.; - extraConfig = '' - proxy_set_header X-Scheme $scheme; - # nginx auth_request includes headers but not body - proxy_set_header Content-Length ""; - proxy_pass_request_body off; - ''; - }; -# locations."/netdata/" = { -# proxyPass = "http://outer:19999/"; -# extraConfig = '' -# auth_request /oauth2/auth; -# error_page 401 = /oauth2/sign_in; -# -# auth_request_set $auth_cookie $upstream_http_set_cookie; -# add_header Set-Cookie $auth_cookie; -# ''; -# }; -# locations."/netdata-ingress/" = { -# proxyPass = "http://outer:19998/"; -# extraConfig = '' -# auth_request /oauth2/auth; -# error_page 401 = /oauth2/sign_in; -# -# auth_request_set $auth_cookie $upstream_http_set_cookie; -# add_header Set-Cookie $auth_cookie; -# ''; -# }; - }; - }; - - - # tootctl on path - environment.systemPackages = [ pkgs.mastodonFork ]; - system.stateVersion = "22.05"; - }; - }; - }; - - config.system.activationScripts.createMastodonContainerPaths = pkgs.lib.stringAfter [ "setupSecrets" "var" ] '' - mkdir -p /var/lib/mastodon-container/mastodon - mkdir -p /var/lib/mastodon-container/redis-mastodon - mkdir -p /var/lib/mastodon-container/postgresql - mkdir -p /var/lib/mastodon-container/elasticsearch - mkdir -p /var/lib/mastodon-container/acme - mkdir -p /var/lib/mastodon-container/certs - mkdir -p /var/lib/mastodon-container/backup - - mkdir -p /var/lib/mastodon-container/secrets - cp "${config.sops.secrets.oauth2_proxy_keys.path}" /var/lib/mastodon-container/secrets/oauth2_proxy_keys - # this happens to be the uid for oauth2_proxy - chown 996 /var/lib/mastodon-container/secrets - chown 996 /var/lib/mastodon-container/secrets/* - ''; + config.system.activationScripts.createMastodonContainerPaths = + pkgs.lib.stringAfter [ "setupSecrets" "var" ] + prodContainer.activationScript; config.sops.secrets.oauth2_proxy_keys = { }; @@ -252,14 +125,13 @@ }) ]; - config.sops.defaultSopsFile = ../secrets/backend.yaml; config.sops.secrets.borg_backup_repo_passphrase = { }; - config.sops.secrets.borgbase_ssh_private_key = { }; # it is extremely important for this to have a trailing newline, or connecting will fail + config.sops.secrets.borgbase_ssh_private_key = + { }; # it is extremely important for this to have a trailing newline, or connecting will fail config.services.borgbackup.jobs."borgbase" = { paths = [ "/var/lib/mastodon-container" "/var/backup" ]; - exclude = [ - ]; + exclude = [ ]; repo = "h5g87o5w@h5g87o5w.repo.borgbase.com:repo"; encryption = { @@ -377,8 +249,9 @@ mastodon: "sidekiq" ''; in { - ExecStart = "${pkgs.prometheus-statsd-exporter}/bin/statsd_exporter --web.listen-address=':9102' --statsd.listen-udp=':9125' --statsd.mapping-config=${mappingFile}"; + ExecStart = + "${pkgs.prometheus-statsd-exporter}/bin/statsd_exporter --web.listen-address=':9102' --statsd.listen-udp=':9125' --statsd.mapping-config=${mappingFile}"; }; }; } - + diff --git a/modules/mastodon_container.nix b/modules/mastodon_container.nix new file mode 100644 index 0000000..36a4f5e --- /dev/null +++ b/modules/mastodon_container.nix @@ -0,0 +1,189 @@ +{ ... }: +{ containerName, hostAddress, domain, localAddress, useElasticsearch +, mastodonPackage, forwardPorts, imports, disabledModules, acme, smtp +, mastodonExtraConfig, oauth2ProxyUsers, oauth2ProxyKeys }: { + + containerConfig = { + #ephemeral = true; + autoStart = true; + privateNetwork = true; + inherit hostAddress; + inherit localAddress; + inherit forwardPorts; + bindMounts = { + "/var/lib/mastodon" = { + hostPath = "/var/lib/${containerName}-container/mastodon"; + isReadOnly = false; + }; + "/var/lib/redis-mastodon" = { + hostPath = "/var/lib/${containerName}-container/redis-mastodon"; + isReadOnly = false; + }; + "/var/lib/postgresql" = { + hostPath = "/var/lib/${containerName}-container/postgresql"; + isReadOnly = false; + }; + "/var/lib/elasticsearch" = { + hostPath = "/var/lib/${containerName}-container/elasticsearch"; + isReadOnly = false; + }; + "/var/lib/acme" = { + hostPath = "/var/lib/${containerName}-container/acme"; + isReadOnly = false; + }; + "/var/lib/certs" = { + hostPath = "/var/lib/${containerName}-container/certs"; + isReadOnly = false; + }; + "/var/lib/secrets" = { + hostPath = "/var/lib/${containerName}-container/secrets"; + isReadOnly = true; + }; + "/var/backup" = { + hostPath = "/var/lib/${containerName}-container/backup"; + isReadOnly = false; + }; + }; + config = { pkgs, ... }: { + inherit imports; + inherit disabledModules; + + networking = { + firewall.enable = true; + firewall.allowedTCPPorts = [ 443 80 ]; + proxy.default = "http://outer:3128"; + proxy.noProxy = "127.0.0.1,localhost,outer,${hostAddress}"; + extraHosts = '' + ${hostAddress} outer + ''; + }; + security = { inherit acme; }; + + services.redis.servers.mastodon = { + enable = true; + bind = "127.0.0.1"; + port = 31637; + }; + + services.mastodon = { + enable = true; + + package = mastodonPackage; + configureNginx = true; + localDomain = domain; + enableUnixSocket = true; + redis = { + createLocally = true; + host = "127.0.0.1"; + port = 31637; + }; + database = { + createLocally = true; + host = "/run/postgresql"; + port = 5432; + }; + inherit smtp; + extraConfig = mastodonExtraConfig; + elasticsearch.host = "127.0.0.1"; + trustedProxy = hostAddress; + }; + + # enable pghero + services.postgresql.settings.shared_preload_libraries = + "pg_stat_statements"; + services.postgresql.settings."pg_stat_statements.track" = "all"; + + services.postgresqlBackup = { + enable = true; + databases = [ "mastodon" ]; + startAt = "*-*-* 04:15:00"; + location = "/var/backup/postgresql"; + }; + + nixpkgs.config.allowUnfree = useElasticsearch; # elasticsearch + services.elasticsearch = { + enable = useElasticsearch; + package = pkgs.elasticsearch7; + extraConf = '' + ingest.geoip.downloader.enabled: false + xpack.security.enabled: false + ''; + }; + + services.oauth2_proxy_mastodon = { + enable = true; + provider = "mastodon"; + keyFile = "/var/lib/secrets/oauth2_proxy_keys"; + setXauthrequest = true; + mastodon = { mastodon-url = "https://${domain}"; }; + extraConfig = { }; + email.addresses = oauth2ProxyUsers; + }; + + services.nginx = { + virtualHosts."${domain}" = { + locations."/oauth2/" = { + proxyPass = + "http://127.0.0.1:4180"; # can't use config.services.oauth2_proxy_mastodon.httpAddress fsr. probably because of weird container module stuff.; + extraConfig = '' + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri; + ''; + }; + locations."/oauth2/auth" = { + proxyPass = + "http://127.0.0.1:4180"; # can't use config.services.oauth2_proxy_mastodon.httpAddress fsr. probably because of weird container module stuff.; + extraConfig = '' + proxy_set_header X-Scheme $scheme; + # nginx auth_request includes headers but not body + proxy_set_header Content-Length ""; + proxy_pass_request_body off; + ''; + }; + # locations."/netdata/" = { + # proxyPass = "http://outer:19999/"; + # extraConfig = '' + # auth_request /oauth2/auth; + # error_page 401 = /oauth2/sign_in; + # + # auth_request_set $auth_cookie $upstream_http_set_cookie; + # add_header Set-Cookie $auth_cookie; + # ''; + # }; + # locations."/netdata-ingress/" = { + # proxyPass = "http://outer:19998/"; + # extraConfig = '' + # auth_request /oauth2/auth; + # error_page 401 = /oauth2/sign_in; + # + # auth_request_set $auth_cookie $upstream_http_set_cookie; + # add_header Set-Cookie $auth_cookie; + # ''; + # }; + }; + }; + + # tootctl on path + environment.systemPackages = [ pkgs.mastodonFork ]; + system.stateVersion = "22.05"; + }; + }; + + activationScript = '' + mkdir -p /var/lib/${containerName}-container/mastodon + mkdir -p /var/lib/${containerName}-container/redis-mastodon + mkdir -p /var/lib/${containerName}-container/postgresql + mkdir -p /var/lib/${containerName}-container/elasticsearch + mkdir -p /var/lib/${containerName}-container/acme + mkdir -p /var/lib/${containerName}-container/certs + mkdir -p /var/lib/${containerName}-container/backup + + mkdir -p /var/lib/${containerName}-container/secrets + cp "${oauth2ProxyKeys}" /var/lib/${containerName}-container/secrets/oauth2_proxy_keys + # this happens to be the uid for oauth2_proxy + chown 996 /var/lib/${containerName}-container/secrets + chown 996 /var/lib/${containerName}-container/secrets/* + ''; + +} +