Add reverse proxy to existing docker host

.gitignore

@@ -4,4 +4,6 @@
 *.tfstate
 *.tfstate.backup
 *.tfstate.*.backup
+*.envrc.private
+.direnv
 result

.sops.yaml

@@ -0,0 +1,13 @@
+keys:
+  - &laptop age1thulhunl9qf552rnlvhrdjrfy3udhfy43389thm5ehr09ycrwcsqdjd25q
+  - &vpn age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6
+  - &vpn_ssh age1gqtj74kr2yumd7wkaf83j2ctlmltv6ykvkwna4thjjmr0v0tts6qnt5dc0
+  - &builder age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6
+creation_rules:
+  - path_regex: targets/vpn/secrets/*
+    key_groups:
+      - age:
+        - *laptop
+        - *vpn
+        - *vpn_ssh
+        - *builder

README.md

@@ -2,19 +2,62 @@
 
 This is an experimental configuration for my Hetzner VPS and Cloudflare to run a VPN using OpenTofu and Nix, based on [NixOS/nixos-wiki-infra on Github](https://github.com/NixOS/nixos-wiki-infra).
 
-## How to use
+## Configure ssh
+
+Add keys to `target/admin/terraform.tf`.
+
+## Configure .env
 
 Copy `.env.example` to `.env` and fill in the values.
 
-To generate a token with Hetzner, go to the project and click `Security -> API Tokens`.
+### Hetzner
 
-For cross-compiling, you will need to add a builder by visiting the following resources:
+In the Hetzner Cloud dashboard, go to the project and click `Security -> API Tokens`.
+
+### Cloudflare
+
+In the Cloudflare user settings, generate an API token with write access to DNS zones.
+
+## Configure sops
+
+On the Terraform client, run:
+
+```bash
+nix develop
+cd targets/vpn/secrets
+just generate-key
+cp secrets.yaml.example secrets.yaml
+```
+
+Populate `secrets.yaml` with the desired values. Run `age-keygen` to get another key specifically for the server and put it in `secrets.yaml` (note that this will only work during installation). Then run `just encrypt` to encrypt and `just decrypt` to decrypt. Put the public key for both the VPN server and the OpenTofu client in `.sops.yaml` in the project root directory.
+
+If you have already installed the server without a key, then run on the server:
+
+```bash
+sudo mkdir -p /var/lib/secrets
+sudo chmod 700 /var/lib/secrets
+sudo chown root:root /var/lib/secrets
+umask 0177
+age-keygen | sudo tee /var/lib/secrets/age >/dev/null
+sudo chmod 600 /var/lib/secrets/age
+sudo chown root:root /var/lib/secrets/age
+umask 0022
+cat /var/lib/secrets/age
+```
+
+Or run on the client: `nix-shell -p ssh-to-age --run 'ssh-keyscan example.com | ssh-to-age'` to get a public age key from the server's public ssh key.
+
+Then follow the above instructions to add the public key.
+
+## Apply configuration
+
+For cross-compiling on different architectures, you will need to add a builder by visiting the following resources:
 - https://nix.dev/tutorials/nixos/distributed-builds-setup.html
 - https://nix.dev/manual/nix/2.25/advanced-topics/distributed-builds
 
 Run `nix develop` at the root of the project directory to access a shell where OpenTofu is accessible.
 
-In the `targets` directory, run `./apply.sh` to update the configurations.
+In the `targets` directory, run `just` to update the configurations.
 
 ## VPN
 
@@ -30,3 +73,11 @@ On the client run:
 ```bash
 tailscale up --login-server <HEADSCALE_URL> --auth-key <KEY>
 ```
+
+## Troubleshooting
+
+### Hetzner VPN
+
+Some stuff may need to be configured by hand. For instance, the network settings may change with a new installation and need to be updated in `targets/vpn/configuration.nix`, or it may cause the network to be disabled requiring mounting with a NixOS recovery image and using `nixos-enter` to redo the networking and rebuild the system.
+
+It may also complain about a hostname change. Changing the name of the VPN in `terraform/nixos-vpn` from `nixos-vpn` to `vpn` might help but I haven't tested it out yet. Otherwise you may need to clone the repo inside of the VM and do `nixos-rebuild` there.

flake.lock

@@ -61,9 +61,30 @@
         "disko": "disko",
         "flake-parts": "flake-parts",
         "nixpkgs": "nixpkgs",
+        "sops-nix": "sops-nix",
         "srvos": "srvos"
       }
     },
+    "sops-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1739262228,
+        "narHash": "sha256-7JAGezJ0Dn5qIyA2+T4Dt/xQgAbhCglh6lzCekTVMeU=",
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "rev": "07af005bb7d60c7f118d9d9f5530485da5d1e975",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "type": "github"
+      }
+    },
     "srvos": {
       "inputs": {
         "nixpkgs": [

flake.nix

@@ -10,6 +10,10 @@
 
     srvos.url = "github:numtide/srvos";
     srvos.inputs.nixpkgs.follows = "nixpkgs";
+
+    sops-nix.url = "github:Mic92/sops-nix";
+    sops-nix.inputs.nixpkgs.follows = "nixpkgs";
+    sops-nix.inputs.nixpkgs-stable.follows = "";
   };
 
   outputs = inputs@{ flake-parts, nixpkgs, ... }:
@@ -37,6 +41,10 @@
             tofuPkg
             pkgs.terraform-ls
             pkgs.hcloud
+            pkgs.age
+            pkgs.sops
+            pkgs.direnv
+            pkgs.just
           ];
 
           shellHook = ''

modules/flake-module.nix

@@ -4,7 +4,11 @@
     hcloud.imports = [
       inputs.srvos.nixosModules.server
       inputs.srvos.nixosModules.hardware-hetzner-cloud
+      inputs.sops-nix.nixosModules.sops
       ./single-disk.nix
+      {
+        sops.age.keyFile = "/var/lib/secrets/age";
+      }
     ];
   };
 }

targets/Justfile

@@ -0,0 +1,6 @@
+default:
+    set -euo pipefail
+    cd "$(dirname "$0")"
+    rm -f .terraform.lock.hcl
+    tofu init
+    tofu apply "$@"

targets/dns/terraform.tf

@@ -15,12 +15,71 @@ variable "vpn_ipv4" {
   description = "IPv4 address for VPN"
 }
 
+variable "vpn_ipv6" {
+  type = string
+  description = "IPv6 address for VPN"
+}
+
 variable "vpn_hostname" {
   type = string
   description = "Hostname for VPN"
 }
 
-resource "cloudflare_record" "vpn" {
+resource "cloudflare_record" "realname_ipv4" {
+  zone_id = module.dns.zone_id_realname
+  name    = module.dns.domain_realname
+  value   = var.vpn_ipv4
+  type    = "A"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "netname_ipv4" {
+  zone_id = module.dns.zone_id_netname
+  name    = module.dns.domain_netname
+  value   = var.vpn_ipv4
+  type    = "A"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "realname_ipv6" {
+  zone_id = module.dns.zone_id_realname
+  name    = module.dns.domain_realname
+  value   = var.vpn_ipv6
+  type    = "AAAA"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "netname_ipv6" {
+  zone_id = module.dns.zone_id_netname
+  name    = module.dns.domain_netname
+  value   = var.vpn_ipv6
+  type    = "AAAA"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "realname_wildcard" {
+  zone_id = module.dns.zone_id_realname
+  name    = "*"
+  value   = module.dns.domain_realname
+  type    = "CNAME"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "netname_wildcard" {
+  zone_id = module.dns.zone_id_netname
+  name    = "*"
+  value   = module.dns.domain_netname
+  type    = "CNAME"
+  ttl     = 3600
+  proxied = false
+}
+
+resource "cloudflare_record" "vpn_ipv4" {
   zone_id = module.dns.zone_id_netname
   name    = "${var.vpn_hostname}.${module.dns.domain_netname}"
   value   = var.vpn_ipv4
@@ -28,3 +87,12 @@ resource "cloudflare_record" "vpn" {
   ttl     = 3600
   proxied = false
 }
+
+resource "cloudflare_record" "vpn_ipv6" {
+  zone_id = module.dns.zone_id_netname
+  name    = "${var.vpn_hostname}.${module.dns.domain_netname}"
+  value   = var.vpn_ipv6
+  type    = "AAAA"
+  ttl     = 3600
+  proxied = false
+}

targets/terraform.tf

@@ -9,6 +9,7 @@ module "vpn" {
 module "dns" {
   source = "./dns"
   vpn_ipv4 = module.vpn.ipv4_address
+  vpn_ipv6 = module.vpn.ipv6_address
   vpn_hostname = module.vpn.hostname
 }
 

targets/vpn/configuration.nix

@@ -31,6 +31,20 @@ in
     };
   };
 
+  sops = {
+    #secrets = {
+    #  cloudflare-api-token = {};
+    #};
+    templates."caddy-env.conf".content = ''
+      CLOUDFLARE_API_TOKEN=${config.sops.placeholder.cloudflare-api-token}
+    '';
+    defaultSopsFile = ./secrets/secrets.yaml;
+    age = {
+      keyFile = "/var/lib/secrets/age";
+      generateKey = true;
+    };
+  };
+
   services = {
     openssh = {
       enable = true;
@@ -41,7 +55,7 @@ in
 
     headscale = {
       enable = true;
-      address = "0.0.0.0";
+      address = "[::]";
       port = 8080;
       settings = {
         server_url = "https://${nixosVars.hostname}.${nixosVars.domain_netname}";
@@ -51,12 +65,15 @@ in
           magic_dns = true;
           search_domains = ["${nixosVars.domain_netname}"];
           nameservers.global = [
-            "1.1.1.1"
             "9.9.9.9"
+            "149.112.112.112"
+            "2620:fe::fe"
+            "2620:fe::9"
           ];
         };
         ip_prefixes = [
           "100.64.0.0/10"
+          "fd7a:115c:a1e0::/48"
         ];
       };
     };
@@ -67,16 +84,58 @@ in
 
     caddy = {
       enable = true;
-      virtualHosts."${nixosVars.hostname}.${nixosVars.domain_netname}".extraConfig = ''
-        reverse_proxy * 127.0.0.1:8080
+      package = pkgs.caddy.withPlugins {
+        plugins = [ "github.com/caddy-dns/cloudflare@v0.0.0-20250214163716-188b4850c0f2" ];
+        hash = "sha256-izuQXvxIq3ycxcUuMErz7MbP9RwLkj+bhliK9H6Heqc=";
+      };
+      globalConfig = ''    
+        acme_dns cloudflare {env.CLOUDFLARE_API_TOKEN}
+      '';
+      virtualHosts = {
+        "${nixosVars.hostname}.${nixosVars.domain_netname}".extraConfig = ''
+          reverse_proxy localhost:8080
+        '';
+        "ts.${nixosVars.domain_netname}".extraConfig = ''
+          respond "Access Denied" 403
+        '';
+        "*.ts.${nixosVars.domain_netname}".extraConfig = ''
+          respond "Access Denied" 403
+        '';
+        "${nixosVars.domain_realname}".extraConfig = ''
+          reverse_proxy http://docker
+        '';
+        "${nixosVars.domain_netname}".extraConfig = ''
+          reverse_proxy http://docker
+        '';
+        "*.${nixosVars.domain_realname}".extraConfig = ''
+          reverse_proxy http://docker
+        '';
+        "*.${nixosVars.domain_netname}".extraConfig = ''
+          reverse_proxy http://docker
         '';
       };
     };
+  };
 
-  systemd.network.networks."10-wan" = {
+  systemd = {
+    services = {
+      caddy = {
+        unitConfig = {
+          After = [ "sops-nix.service" ];
+        };
+        serviceConfig = {
+          EnvironmentFile = lib.mkForce [config.sops.templates."caddy-env.conf".path];
+        };
+      };
+    };
+    network.networks."10-wan" = {
   	  matchConfig.MACAddress = "96:00:04:16:ed:c5";
-    address = ["${nixosVars.ipv4_address}/32"];
+      address = [
+        "${nixosVars.ipv4_address}/32"
+        "${nixosVars.ipv6_address}/64"
+      ];
       routes = [
+        { Gateway = "fe80::1"; }
         {
           Gateway = "172.31.1.1";
           GatewayOnLink = true;
@@ -84,6 +143,7 @@ in
       ];
       linkConfig.RequiredForOnline = "routable";
     };
+  };
 
   boot.supportedFilesystems = ["btrfs"];
   environment.systemPackages = [
@@ -94,6 +154,7 @@ in
     pkgs.git
     pkgs.hcloud
     pkgs.dhcpcd
+    pkgs.age
   ];
 
   boot.kernel.sysctl = {

targets/vpn/deploy.sh

@@ -10,6 +10,6 @@ nixBuild() {
   fi
 }
 nixBuild .#nixosConfigurations.vpn.config.system.build.toplevel -L
-if ! nixos-rebuild switch --flake .#vpn --target-host root@vpn; then
-  nixos-rebuild switch --flake .#vpn --target-host root@vpn
+if ! nixos-rebuild switch --flake .#vpn --target-host root@vpn --impure; then
+  nixos-rebuild switch --flake .#vpn --target-host root@vpn --impure
 fi

targets/vpn/secrets/Justfile

@@ -0,0 +1,13 @@
+default:
+
+generate-key:
+    mkdir -p ~/.config/sops/age
+    age-keygen -o ~/.config/sops/age/keys.txt
+    cat ~/.config/sops/age/keys.txt
+
+encrypt:
+    sops --encrypt --in-place secrets.yaml
+
+decrypt:
+    sops --decrypt --in-place secrets.yaml
+

targets/vpn/secrets/secrets.yaml

@@ -0,0 +1,40 @@
+age-key: ENC[AES256_GCM,data:s1aKQoHd8HX5I+1LrQao8wQbw7efUFmq2isAw/oGglbKGFLrTG3zHvO4NeduzG6NBb8ZmYLiWmfGJdkG3w7Qm7msIU48RWF0jD0=,iv:nZFlqLPL7A+APGuWwljqBQlauMZZxB2//4OhMM2RolU=,tag:gaxry+cHfgAoj41DyJLvSw==,type:str]
+cloudflare-api-token: ENC[AES256_GCM,data:W7eZldBRdPsuOM6OHlUfR7itTFnkzYe3FO+Z6C1fqnH7lcAHna3OxA==,iv:REhJNt07qP05xFYEfyQbxou+ciRfeol9HDJwbj8yM3Q=,tag:qvbWmCnyfI0CKYfjMz53MA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1thulhunl9qf552rnlvhrdjrfy3udhfy43389thm5ehr09ycrwcsqdjd25q
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVbGdoelVNb3NNcThqV2Nl
+            c2VVaTJqQTlCdHlMaktjVW1hYUNpVThKK0NrCng5cmtiLzIvVUhPNUs5ZVl6Q0FN
+            Q21UUHBIVXkzcTh4RC8zNmplKzREdlkKLS0tIHdNZmlZUU1NMWZNTk9GT3dXejFM
+            NzJVZDRZc1BOQU40b2VwdnhmTVdGdk0KImC16x+U77eqwFnYpmCB6xFVgvRaWCw8
+            WwFL3NG9Ex2BEHjWr7/WGSnPXw5y5wWdBRoO5D/Yp8txPmWdRxzJLg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoWGNIcGR2VHd5S1YvODhT
+            Q0xMR3RxUjc4TWZQaHlkdTZtWmJTRHh1Q0dnCktkV3E1QS9acklraXIxR1NEdE0x
+            djZvSkNIVXVNVENtZGlFSVhlQ0JLakEKLS0tIElvU1plM3RvSEFWN05JZ0ZqbnBB
+            OTNjR0lDQkEzYmhRTG52cGFQdExNSm8K2tSemLa8RRGTfSfMi15HaeAJLo7512aw
+            u4JtCMkUOfDB61wt1ibWgasfUcyUWk8d3DCaz5ruKU11WfkDi1f+5g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1gqtj74kr2yumd7wkaf83j2ctlmltv6ykvkwna4thjjmr0v0tts6qnt5dc0
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYVTJOR2xjUlZQZEViTlhn
+            NnExSm9jeTlOZEZGNzVreVVsK0drS0ozLzBrCjg1SmhjMloxaGVPUFIxNndJVFNI
+            STRPaFY2M0xxNDY5VTZtNlc3a0k5R3MKLS0tIG1rczRrUWd2UmtsTm81RURVYjVo
+            VlhVSjhUeGI4WDlCQXJXaUdGdld2UE0KJGzt/0tQBE4LPKuTh67hi/P3vp+dzH4B
+            dqzYs8SgwJDMofEDA9b5+FG3tixFnGf6UATz3W2ZQK/WOMchAH5/zw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2025-02-23T03:41:06Z"
+    mac: ENC[AES256_GCM,data:tM1snUKWYf7CNfg/OwXg/VM334wgqhAfr12Fhka0OiDhFYeXRSTwUcM9U3rByZ9hBdT89q94svioVSxYpc74RGXj+iuX1fVluEdg8KtNSsnw2R351OGQ42ijeeKcGKETTs7zqQMnZM85xPF2tmc2QYj6uPih0UPISYdSzdB561s=,iv:e8RL61ajNASdjrcHnFd3TG1VFgXoj1P/BvwkYSlAfKA=,tag:e767zoOAIwCSYyvWUvABUg==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.9.4

targets/vpn/secrets/secrets.yaml.example

@@ -0,0 +1,2 @@
+age-key: age1zyx...
+cloudflare-api-token: d4d...

targets/vpn/terraform.tf

@@ -2,6 +2,7 @@ module "vpn" {
   source           = "../../terraform/nixos-vpn"
   nixos_flake_attr = "vpn"
   nixos_vars_file  = "${path.module}/nixos-vars.json"
+  sops_file        = abspath("${path.module}/secrets/secrets.yaml")
   tags = {
     Terraform = "true"
     Target    = "vpn"

terraform/nixos-vpn/decrypt-age-keys.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail -x
+
+mkdir -p var/lib/secrets
+
+umask 0177
+sops --extract '["age-key"]' --decrypt "$SOPS_FILE" >./var/lib/secrets/age
+# restore umask
+umask 0022

terraform/nixos-vpn/main.tf

@@ -28,6 +28,10 @@ module "deploy" {
   target_host            = hcloud_server.nixos_vpn.ipv4_address
   instance_id            = hcloud_server.nixos_vpn.id
   debug_logging          = true
+  extra_files_script     = "${path.module}/decrypt-age-keys.sh"
+  extra_environment = {
+    SOPS_FILE = var.sops_file
+  }
 }
 
 locals {

terraform/nixos-vpn/variables.tf

@@ -21,6 +21,11 @@ variable "nixos_vars_file" {
   description = "File to write NixOS configuration variables to"
 }
 
+variable "sops_file" {
+  type        = string
+  description = "File to SOPS secrets file"
+}
+
 variable "nixos_flake_attr" {
   type        = string
   description = "NixOS configuration flake attribute"