Keyboard shortcuts

Press ← or → to navigate between chapters

Press S or / to search in the book

Press ? to show this help

Press Esc to hide this help

aerialis

This is one of the two main infrastructure hosts (see also: stormwing). All services and containers for aerialis are defined in nixos/hosts/aerialis.nix and its submodules.

Host configuration

{
  garuda-lib,
  ...
}:
let
  mon = garuda-lib.monitoring;
in
{
  imports = [
    ../modules
    ./../modules/special/hetzner-ex44.nix
  ];

  garuda =
    garuda-lib.mkMonitoring {
      host = "aerialis";
      hostInfo = true;
      units = [
        "borgmatic.service"
        "borgmatic.timer"
        "sshd.service"
        "tailscaled.service"
      ];
      exporters = [
        "smartctlExporter"
        "borgmaticExporter"
      ];
    }
    // {
      backup.borgmatic = {
        enable = true;
        name = "aerialis";
        user = "u342919";
        host = "u342919.your-storagebox.de";
        port = 23;
        label = "hetzner";
        sshKeySecret = "backup/ssh_aerialis";
        knownHosts = [
          {
            name = "storagebox-ed25519";
            publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs"; # pragma: allowlist secret
          }
          {
            name = "storagebox-rsa";
            publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw=="; # pragma: allowlist secret
          }
          {
            name = "storagebox-ecdsa";
            publicKey = "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAGK0po6usux4Qv2d8zKZN1dDvbWjxKkGsx7XwFdSUCnF19Q8psHEUWR7C/LtSQ5crU/g+tQVRBtSgoUcE8T+FWp5wBxKvWG2X9gD+s9/4zRmDeSJR77W6gSA/+hpOZoSE+4KgNdnbYSNtbZH/dN74EG7GLb/gcIpbUUzPNXpfKl7mQitw=="; # pragma: allowlist secret
          }
        ];
        sourceDirectories = [
          "/data_1/containers"
          "/data_1/persistent/etc/ssh"
          "/data_2/backup/nextcloud-aio/"
          "/data_2/containers/chaotic-backend/chaotic/database"
          "/data_2/containers/mastodon"
          "/data_2/containers/postgres"
        ];
        excludePatterns = [
          "/data_1/dockercache"
          "/data_1/dockerdata"
        ];
      };
    };

  networking = {
    defaultGateway = "157.180.57.65";
    defaultGateway6 = {
      address = "fe80::1";
      interface = "eth0";
    };
    hostName = "aerialis";
    interfaces."eth0".ipv4.addresses = [
      {
        address = "157.180.57.100";
        prefixLength = 26;
      }
    ];
    nat.forwardPorts = [
      (garuda-lib.mkNatForward {
        sourcePort = 80;
        destination = "10.0.5.10:80";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 443;
        destination = "10.0.5.10:443";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 443;
        destination = "10.0.5.10:443";
        proto = "udp";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 465;
        destination = "10.0.5.80:465";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 993;
        destination = "10.0.5.80:993";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 25;
        destination = "10.0.5.80:25";
      })
      (garuda-lib.mkNatForward {
        sourcePort = 4190;
        destination = "10.0.5.80:4190";
      })
    ];
  };

  # Can't set this inside the containers
  boot.kernel.sysctl."vm.overcommit_memory" = "1";

  services.garuda-nspawn = {
    dockerCache = "/data_1/dockercache/";

    containers = garuda-lib.mkContainers {
      dir = ./aerialis;
      ips = mon.aerialisContainers;
      containers = {
        chaotic-backend = {
          mounts = [
            {
              name = "chaotic";
              hostPath = "/data_2/containers/chaotic-backend/chaotic";
              mountPoint = "/var/garuda/compose-runner/chaotic-backend";
            }
            {
              name = "chaotic-redis";
              hostPath = "/data_2/containers/chaotic-backend/redis";
              mountPoint = "/var/lib/redis-chaotic";
            }
          ];
          forwardPorts = [
            {
              containerPort = 22;
              hostPort = 270;
            }
          ];
          nspawn = {
            enableTun = true;
          };
          needsDocker = true;
        };

        docker = {
          mounts = [
            {
              name = "compose";
              hostPath = "/data_1/containers/docker/";
              mountPoint = "/var/garuda/compose-runner/docker";
            }
            {
              name = "nextcloud-local-backup";
              hostPath = "/data_2/backup/nextcloud-aio";
              mountPoint = "/var/garuda/backups/nextcloud";
            }
          ];
          needsDocker = true;
        };

        docker-proxied = {
          mounts = [
            {
              name = "compose";
              hostPath = "/data_1/containers/docker-proxied/";
              mountPoint = "/var/garuda/compose-runner/docker-proxied";
            }
          ];
          needsDocker = true;
        };

        forum = {
          mounts = [
            {
              name = "forum";
              hostPath = "/data_1/containers/forum/";
              mountPoint = "/var/discourse";
            }
          ];
          needsDocker = true;
        };

        mastodon = {
          mounts = [
            {
              name = "mastodon";
              hostPath = "/data_2/containers/mastodon/mastodon";
              mountPoint = "/var/lib/mastodon";
            }
            {
              name = "compose";
              hostPath = "/data_1/containers/mastodon/compose";
              mountPoint = "/var/garuda/compose-runner/mastodon";
            }
          ];
          needsDocker = true;
        };

        mail = {
          mounts = [
            {
              name = "acme";
              hostPath = "/data_2/containers/web-front/acme";
              mountPoint = "/var/lib/acme";
            }
            {
              name = "dkim";
              hostPath = "/data_1/containers/mail/dkim";
              mountPoint = "/var/dkim";
            }
            {
              name = "index";
              hostPath = "/data_1/containers/mail/index";
              mountPoint = "/var/lib/dovecot/indices";
            }
            {
              name = "rspamd";
              hostPath = "/data_1/containers/mail/rspamd";
              mountPoint = "/var/lib/redis-rspamd";
            }
            {
              name = "vmail";
              hostPath = "/data_1/containers/mail/vmail";
              mountPoint = "/var/vmail";
            }
          ];
        };

        postgres = {
          mounts = [
            {
              name = "data";
              hostPath = "/data_1/containers/postgres/data";
              mountPoint = "/var/lib/postgresql";
            }
            {
              name = "postgres_backup";
              hostPath = "/data_2/containers/postgres/backup";
              mountPoint = "/var/garuda/backups/postgres";
            }
            {
              name = "acme";
              hostPath = "/data_2/containers/web-front/acme";
              mountPoint = "/var/lib/acme";
              readOnly = true;
            }
          ];
          forwardPorts = [
            {
              containerPort = 22;
              hostPort = 220;
            }
            { containerPort = 5432; }
          ];
        };

        web-front = {
          mounts = [
            {
              name = "acme";
              hostPath = "/data_2/containers/web-front/acme";
              mountPoint = "/var/lib/acme";
            }
            {
              name = "nginx";
              hostPath = "/data_2/containers/web-front/nginx";
              mountPoint = "/var/log/nginx";
            }
          ];
          forwardPorts = [
            {
              containerPort = 22;
              hostPort = 210;
            }
          ];
        };

        n8n = {
          mounts = [
            {
              name = "n8n";
              hostPath = "/data_1/containers/n8n/var-lib";
              mountPoint = "/var/lib";
            }
          ];
        };

        monitoring = {
          mounts = [
            {
              name = "alertmanager";
              hostPath = "/data_1/containers/monitoring/alertmanager";
              mountPoint = "/var/lib/private/alertmanager";
            }
            {
              name = "prometheus";
              hostPath = "/data_1/containers/monitoring/prometheus";
              mountPoint = "/var/lib/prometheus2";
            }
            {
              name = "grafana";
              hostPath = "/data_1/containers/monitoring/grafana";
              mountPoint = "/var/lib/grafana";
            }
            {
              name = "loki";
              hostPath = "/data_1/containers/monitoring/loki";
              mountPoint = "/var/lib/loki";
            }
            {
              name = "tailscale";
              hostPath = "/data_1/containers/monitoring/tailscale";
              mountPoint = "/var/lib/tailscale";
            }
          ];
          nspawn = {
            enableTun = true;
          };
        };
      };
    };
  };

  systemd.services =
    garuda-lib.mkContainerOrdering {
      root = "postgres";
      containers = [
        "docker-proxied"
        "docker"
        "mastodon"
        "chaotic-backend"
      ];
    }
    // {
      loki-tailnet-proxy = garuda-lib.mkTunnel {
        bind = mon.tailnetIPs.aerialis;
        listen = mon.ports.loki;
        target = "${mon.loki.containerAddress}:${toString mon.ports.loki}";
        after = [
          "tailscaled.service"
          "[email protected]"
        ];
        wants = [ "[email protected]" ];
        description = "Expose Loki to Tailnet only";
      };
    };

  networking.firewall.interfaces."${mon.bridge.interface}".allowedTCPPorts = [
    mon.ports.nodeExporter
    mon.ports.smartctlExporter
    mon.ports.borgmaticExporter
  ];
}

Shared host configuration

The disk layout, SSH configuration, container bridge (br0) and impermanence are shared with stormwing and live in nixos/modules/special/newgen.nix. Hardware-specific settings (kernel, microcode, SMT, smartd) come from nixos/modules/special/hetzner-ex44.nix.

_: {
  # We want to have same UID's in all containers to allow sharing home directories
  garuda-lib.unifiedUID = true;

  fileSystems."/" = {
    device = "none";
    fsType = "tmpfs";
    options = [
      "defaults"
      "size=50%"
      "mode=755"
    ];
  };

  fileSystems."/data_1" = {
    device = "/dev/disk/by-label/NIXROOT";
    fsType = "ext4";
    neededForBoot = true;
    options = [
      "defaults"
      "noatime"
      "nodiratime"
      "errors=remount-ro"
    ];
    depends = [ "/" ];
  };

  fileSystems."/data_2" = {
    device = "/dev/disk/by-label/NIXDATA";
    fsType = "btrfs";
    options = [
      "defaults"
      "noatime"
      "nodiratime"
      "compress=zstd:1"
    ];
  };

  fileSystems."/boot" = {
    device = "/dev/disk/by-label/NIXBOOT";
    fsType = "vfat";
  };

  fileSystems."/nix" = {
    device = "/data_1/nix";
    fsType = "none";
    options = [ "bind" ];
    depends = [
      "/data_1"
    ];
  };

  services.openssh = {
    ports = [ 666 ];
    hostKeys = [
      {
        type = "ed25519";
        path = "/data_1/persistent/etc/ssh/ssh_host_ed25519_key";
      }
      {
        type = "rsa";
        bits = 4096;
        path = "/data_1/persistent/etc/ssh/ssh_host_rsa_key";
      }
    ];
  };

  services.garuda-nspawn = {
    bridgeInterface = "br0";
    hostInterface = "eth0";
    hostIp = "10.0.5.1";
    defaults = {
      maxMemorySoft = 48318382080; # 45 GiB
      maxMemoryHard = 53687091200; # 50 GiB
      maxCpu = 18;
    };
  };

  networking.firewall.trustedInterfaces = [ "br0" ];

  garuda-lib.sshkeys = {
    ed25519 = "/data_1/persistent/etc/ssh/ssh_host_ed25519_key";
    rsa = "/data_1/persistent/etc/ssh/ssh_host_rsa_key";
  };

  environment.persistence."/data_1/persistent" = {
    enable = true;
    hideMounts = true;
    directories = [
      "/home"
      "/var/cache/netdata"
      "/var/cache/tailscale"
      "/var/lib/netdata"
      "/var/lib/nixos"
      "/var/lib/tailscale"
      "/var/lib/vnstat"
      "/var/log"
    ];
    files = [
      "/etc/machine-id"
    ];
  };

  security.sudo.extraConfig = ''
    Defaults lecture = never
  '';
}

Containers/services

  • chaotic-backend: Backend services for Chaotic-AUR, including API and job processing.
  • docker: General-purpose Docker container runner for services not packaged in Nix.
  • docker-proxied: Docker runner for services that require special proxying or network setup.
  • forum: Hosts the Discourse forum for the Garuda Linux community.
  • mail: Handles mail-related services and relays for the infrastructure.
  • mastodon: Runs the Mastodon social network instance.
  • monitoring: Runs the monitoring stack (Prometheus, Grafana, Loki, Alertmanager).
  • n8n: Workflow automation and incoming webhooks.
  • postgres: Provides PostgreSQL database services for other containers.
  • web-front: Acts as the main reverse proxy and web frontend for hosted services.

See the respective documentation pages for up-to-date configuration and details.