diff --git a/.gitignore b/.gitignore index af23183..107f6c0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ *.tfstate *.tfstate.backup *.tfstate.*.backup +*.envrc.private +.direnv result diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..40e4594 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,13 @@ +keys: + - &laptop age1thulhunl9qf552rnlvhrdjrfy3udhfy43389thm5ehr09ycrwcsqdjd25q + - &vpn age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6 + - &vpn_ssh age1gqtj74kr2yumd7wkaf83j2ctlmltv6ykvkwna4thjjmr0v0tts6qnt5dc0 + - &builder age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6 +creation_rules: + - path_regex: targets/vpn/secrets/* + key_groups: + - age: + - *laptop + - *vpn + - *vpn_ssh + - *builder diff --git a/README.md b/README.md index 4a51729..04bce68 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,62 @@ This is an experimental configuration for my Hetzner VPS and Cloudflare to run a VPN using OpenTofu and Nix, based on [NixOS/nixos-wiki-infra on Github](https://github.com/NixOS/nixos-wiki-infra). -## How to use +## Configure ssh + +Add keys to `target/admin/terraform.tf`. + +## Configure .env Copy `.env.example` to `.env` and fill in the values. -To generate a token with Hetzner, go to the project and click `Security -> API Tokens`. +### Hetzner -For cross-compiling, you will need to add a builder by visiting the following resources: +In the Hetzner Cloud dashboard, go to the project and click `Security -> API Tokens`. + +### Cloudflare + +In the Cloudflare user settings, generate an API token with write access to DNS zones. + +## Configure sops + +On the Terraform client, run: + +```bash +nix develop +cd targets/vpn/secrets +just generate-key +cp secrets.yaml.example secrets.yaml +``` + +Populate `secrets.yaml` with the desired values. Run `age-keygen` to get another key specifically for the server and put it in `secrets.yaml` (note that this will only work during installation). Then run `just encrypt` to encrypt and `just decrypt` to decrypt. Put the public key for both the VPN server and the OpenTofu client in `.sops.yaml` in the project root directory. + +If you have already installed the server without a key, then run on the server: + +```bash +sudo mkdir -p /var/lib/secrets +sudo chmod 700 /var/lib/secrets +sudo chown root:root /var/lib/secrets +umask 0177 +age-keygen | sudo tee /var/lib/secrets/age >/dev/null +sudo chmod 600 /var/lib/secrets/age +sudo chown root:root /var/lib/secrets/age +umask 0022 +cat /var/lib/secrets/age +``` + +Or run on the client: `nix-shell -p ssh-to-age --run 'ssh-keyscan example.com | ssh-to-age'` to get a public age key from the server's public ssh key. + +Then follow the above instructions to add the public key. + +## Apply configuration + +For cross-compiling on different architectures, you will need to add a builder by visiting the following resources: - https://nix.dev/tutorials/nixos/distributed-builds-setup.html - https://nix.dev/manual/nix/2.25/advanced-topics/distributed-builds Run `nix develop` at the root of the project directory to access a shell where OpenTofu is accessible. -In the `targets` directory, run `./apply.sh` to update the configurations. +In the `targets` directory, run `just` to update the configurations. ## VPN @@ -30,3 +73,11 @@ On the client run: ```bash tailscale up --login-server --auth-key ``` + +## Troubleshooting + +### Hetzner VPN + +Some stuff may need to be configured by hand. For instance, the network settings may change with a new installation and need to be updated in `targets/vpn/configuration.nix`, or it may cause the network to be disabled requiring mounting with a NixOS recovery image and using `nixos-enter` to redo the networking and rebuild the system. + +It may also complain about a hostname change. Changing the name of the VPN in `terraform/nixos-vpn` from `nixos-vpn` to `vpn` might help but I haven't tested it out yet. Otherwise you may need to clone the repo inside of the VM and do `nixos-rebuild` there. diff --git a/flake.lock b/flake.lock index 5f1b477..9741ac0 100644 --- a/flake.lock +++ b/flake.lock @@ -61,9 +61,30 @@ "disko": "disko", "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", + "sops-nix": "sops-nix", "srvos": "srvos" } }, + "sops-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1739262228, + "narHash": "sha256-7JAGezJ0Dn5qIyA2+T4Dt/xQgAbhCglh6lzCekTVMeU=", + "owner": "Mic92", + "repo": "sops-nix", + "rev": "07af005bb7d60c7f118d9d9f5530485da5d1e975", + "type": "github" + }, + "original": { + "owner": "Mic92", + "repo": "sops-nix", + "type": "github" + } + }, "srvos": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index f59fed3..8c0073a 100644 --- a/flake.nix +++ b/flake.nix @@ -10,6 +10,10 @@ srvos.url = "github:numtide/srvos"; srvos.inputs.nixpkgs.follows = "nixpkgs"; + + sops-nix.url = "github:Mic92/sops-nix"; + sops-nix.inputs.nixpkgs.follows = "nixpkgs"; + sops-nix.inputs.nixpkgs-stable.follows = ""; }; outputs = inputs@{ flake-parts, nixpkgs, ... }: @@ -37,6 +41,10 @@ tofuPkg pkgs.terraform-ls pkgs.hcloud + pkgs.age + pkgs.sops + pkgs.direnv + pkgs.just ]; shellHook = '' diff --git a/modules/flake-module.nix b/modules/flake-module.nix index 4133449..8a873cb 100644 --- a/modules/flake-module.nix +++ b/modules/flake-module.nix @@ -4,7 +4,11 @@ hcloud.imports = [ inputs.srvos.nixosModules.server inputs.srvos.nixosModules.hardware-hetzner-cloud + inputs.sops-nix.nixosModules.sops ./single-disk.nix + { + sops.age.keyFile = "/var/lib/secrets/age"; + } ]; }; } diff --git a/targets/Justfile b/targets/Justfile new file mode 100644 index 0000000..a17b1bd --- /dev/null +++ b/targets/Justfile @@ -0,0 +1,6 @@ +default: + set -euo pipefail + cd "$(dirname "$0")" + rm -f .terraform.lock.hcl + tofu init + tofu apply "$@" diff --git a/targets/dns/terraform.tf b/targets/dns/terraform.tf index 5937ee2..5b156e2 100644 --- a/targets/dns/terraform.tf +++ b/targets/dns/terraform.tf @@ -15,12 +15,71 @@ variable "vpn_ipv4" { description = "IPv4 address for VPN" } +variable "vpn_ipv6" { + type = string + description = "IPv6 address for VPN" +} + variable "vpn_hostname" { type = string description = "Hostname for VPN" } -resource "cloudflare_record" "vpn" { +resource "cloudflare_record" "realname_ipv4" { + zone_id = module.dns.zone_id_realname + name = module.dns.domain_realname + value = var.vpn_ipv4 + type = "A" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "netname_ipv4" { + zone_id = module.dns.zone_id_netname + name = module.dns.domain_netname + value = var.vpn_ipv4 + type = "A" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "realname_ipv6" { + zone_id = module.dns.zone_id_realname + name = module.dns.domain_realname + value = var.vpn_ipv6 + type = "AAAA" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "netname_ipv6" { + zone_id = module.dns.zone_id_netname + name = module.dns.domain_netname + value = var.vpn_ipv6 + type = "AAAA" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "realname_wildcard" { + zone_id = module.dns.zone_id_realname + name = "*" + value = module.dns.domain_realname + type = "CNAME" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "netname_wildcard" { + zone_id = module.dns.zone_id_netname + name = "*" + value = module.dns.domain_netname + type = "CNAME" + ttl = 3600 + proxied = false +} + +resource "cloudflare_record" "vpn_ipv4" { zone_id = module.dns.zone_id_netname name = "${var.vpn_hostname}.${module.dns.domain_netname}" value = var.vpn_ipv4 @@ -28,3 +87,12 @@ resource "cloudflare_record" "vpn" { ttl = 3600 proxied = false } + +resource "cloudflare_record" "vpn_ipv6" { + zone_id = module.dns.zone_id_netname + name = "${var.vpn_hostname}.${module.dns.domain_netname}" + value = var.vpn_ipv6 + type = "AAAA" + ttl = 3600 + proxied = false +} diff --git a/targets/terraform.tf b/targets/terraform.tf index 09ff4f4..311a3c1 100644 --- a/targets/terraform.tf +++ b/targets/terraform.tf @@ -9,6 +9,7 @@ module "vpn" { module "dns" { source = "./dns" vpn_ipv4 = module.vpn.ipv4_address + vpn_ipv6 = module.vpn.ipv6_address vpn_hostname = module.vpn.hostname } diff --git a/targets/vpn/configuration.nix b/targets/vpn/configuration.nix index bea7373..154ff35 100644 --- a/targets/vpn/configuration.nix +++ b/targets/vpn/configuration.nix @@ -31,6 +31,20 @@ in }; }; + sops = { + #secrets = { + # cloudflare-api-token = {}; + #}; + templates."caddy-env.conf".content = '' + CLOUDFLARE_API_TOKEN=${config.sops.placeholder.cloudflare-api-token} + ''; + defaultSopsFile = ./secrets/secrets.yaml; + age = { + keyFile = "/var/lib/secrets/age"; + generateKey = true; + }; + }; + services = { openssh = { enable = true; @@ -41,7 +55,7 @@ in headscale = { enable = true; - address = "0.0.0.0"; + address = "[::]"; port = 8080; settings = { server_url = "https://${nixosVars.hostname}.${nixosVars.domain_netname}"; @@ -51,12 +65,15 @@ in magic_dns = true; search_domains = ["${nixosVars.domain_netname}"]; nameservers.global = [ - "1.1.1.1" "9.9.9.9" + "149.112.112.112" + "2620:fe::fe" + "2620:fe::9" ]; }; ip_prefixes = [ "100.64.0.0/10" + "fd7a:115c:a1e0::/48" ]; }; }; @@ -67,22 +84,65 @@ in caddy = { enable = true; - virtualHosts."${nixosVars.hostname}.${nixosVars.domain_netname}".extraConfig = '' - reverse_proxy * 127.0.0.1:8080 + package = pkgs.caddy.withPlugins { + plugins = [ "github.com/caddy-dns/cloudflare@v0.0.0-20250214163716-188b4850c0f2" ]; + hash = "sha256-izuQXvxIq3ycxcUuMErz7MbP9RwLkj+bhliK9H6Heqc="; + }; + globalConfig = '' + acme_dns cloudflare {env.CLOUDFLARE_API_TOKEN} ''; + virtualHosts = { + "${nixosVars.hostname}.${nixosVars.domain_netname}".extraConfig = '' + reverse_proxy localhost:8080 + ''; + "ts.${nixosVars.domain_netname}".extraConfig = '' + respond "Access Denied" 403 + ''; + "*.ts.${nixosVars.domain_netname}".extraConfig = '' + respond "Access Denied" 403 + ''; + "${nixosVars.domain_realname}".extraConfig = '' + reverse_proxy http://docker + ''; + "${nixosVars.domain_netname}".extraConfig = '' + reverse_proxy http://docker + ''; + "*.${nixosVars.domain_realname}".extraConfig = '' + reverse_proxy http://docker + ''; + "*.${nixosVars.domain_netname}".extraConfig = '' + reverse_proxy http://docker + ''; + }; }; }; - systemd.network.networks."10-wan" = { - matchConfig.MACAddress = "96:00:04:16:ed:c5"; - address = ["${nixosVars.ipv4_address}/32"]; - routes = [ - { - Gateway = "172.31.1.1"; - GatewayOnLink = true; - } - ]; - linkConfig.RequiredForOnline = "routable"; + systemd = { + services = { + caddy = { + unitConfig = { + After = [ "sops-nix.service" ]; + }; + serviceConfig = { + EnvironmentFile = lib.mkForce [config.sops.templates."caddy-env.conf".path]; + }; + }; + }; + network.networks."10-wan" = { + matchConfig.MACAddress = "96:00:04:16:ed:c5"; + address = [ + "${nixosVars.ipv4_address}/32" + "${nixosVars.ipv6_address}/64" + ]; + routes = [ + { Gateway = "fe80::1"; } + { + Gateway = "172.31.1.1"; + GatewayOnLink = true; + } + ]; + linkConfig.RequiredForOnline = "routable"; + }; }; boot.supportedFilesystems = ["btrfs"]; @@ -94,6 +154,7 @@ in pkgs.git pkgs.hcloud pkgs.dhcpcd + pkgs.age ]; boot.kernel.sysctl = { diff --git a/targets/vpn/deploy.sh b/targets/vpn/deploy.sh index 00928ca..2301012 100755 --- a/targets/vpn/deploy.sh +++ b/targets/vpn/deploy.sh @@ -10,6 +10,6 @@ nixBuild() { fi } nixBuild .#nixosConfigurations.vpn.config.system.build.toplevel -L -if ! nixos-rebuild switch --flake .#vpn --target-host root@vpn; then - nixos-rebuild switch --flake .#vpn --target-host root@vpn +if ! nixos-rebuild switch --flake .#vpn --target-host root@vpn --impure; then + nixos-rebuild switch --flake .#vpn --target-host root@vpn --impure fi diff --git a/targets/vpn/secrets/Justfile b/targets/vpn/secrets/Justfile new file mode 100644 index 0000000..217070a --- /dev/null +++ b/targets/vpn/secrets/Justfile @@ -0,0 +1,13 @@ +default: + +generate-key: + mkdir -p ~/.config/sops/age + age-keygen -o ~/.config/sops/age/keys.txt + cat ~/.config/sops/age/keys.txt + +encrypt: + sops --encrypt --in-place secrets.yaml + +decrypt: + sops --decrypt --in-place secrets.yaml + diff --git a/targets/vpn/secrets/secrets.yaml b/targets/vpn/secrets/secrets.yaml new file mode 100644 index 0000000..ac51638 --- /dev/null +++ b/targets/vpn/secrets/secrets.yaml @@ -0,0 +1,40 @@ +age-key: ENC[AES256_GCM,data:s1aKQoHd8HX5I+1LrQao8wQbw7efUFmq2isAw/oGglbKGFLrTG3zHvO4NeduzG6NBb8ZmYLiWmfGJdkG3w7Qm7msIU48RWF0jD0=,iv:nZFlqLPL7A+APGuWwljqBQlauMZZxB2//4OhMM2RolU=,tag:gaxry+cHfgAoj41DyJLvSw==,type:str] +cloudflare-api-token: ENC[AES256_GCM,data:W7eZldBRdPsuOM6OHlUfR7itTFnkzYe3FO+Z6C1fqnH7lcAHna3OxA==,iv:REhJNt07qP05xFYEfyQbxou+ciRfeol9HDJwbj8yM3Q=,tag:qvbWmCnyfI0CKYfjMz53MA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1thulhunl9qf552rnlvhrdjrfy3udhfy43389thm5ehr09ycrwcsqdjd25q + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVbGdoelVNb3NNcThqV2Nl + c2VVaTJqQTlCdHlMaktjVW1hYUNpVThKK0NrCng5cmtiLzIvVUhPNUs5ZVl6Q0FN + Q21UUHBIVXkzcTh4RC8zNmplKzREdlkKLS0tIHdNZmlZUU1NMWZNTk9GT3dXejFM + NzJVZDRZc1BOQU40b2VwdnhmTVdGdk0KImC16x+U77eqwFnYpmCB6xFVgvRaWCw8 + WwFL3NG9Ex2BEHjWr7/WGSnPXw5y5wWdBRoO5D/Yp8txPmWdRxzJLg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1emavxf6jydt0f8nt7y5xyagthhh0hcc3f0kthtt2yx0am7df3vdqw7uwk6 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoWGNIcGR2VHd5S1YvODhT + Q0xMR3RxUjc4TWZQaHlkdTZtWmJTRHh1Q0dnCktkV3E1QS9acklraXIxR1NEdE0x + djZvSkNIVXVNVENtZGlFSVhlQ0JLakEKLS0tIElvU1plM3RvSEFWN05JZ0ZqbnBB + OTNjR0lDQkEzYmhRTG52cGFQdExNSm8K2tSemLa8RRGTfSfMi15HaeAJLo7512aw + u4JtCMkUOfDB61wt1ibWgasfUcyUWk8d3DCaz5ruKU11WfkDi1f+5g== + -----END AGE ENCRYPTED FILE----- + - recipient: age1gqtj74kr2yumd7wkaf83j2ctlmltv6ykvkwna4thjjmr0v0tts6qnt5dc0 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYVTJOR2xjUlZQZEViTlhn + NnExSm9jeTlOZEZGNzVreVVsK0drS0ozLzBrCjg1SmhjMloxaGVPUFIxNndJVFNI + STRPaFY2M0xxNDY5VTZtNlc3a0k5R3MKLS0tIG1rczRrUWd2UmtsTm81RURVYjVo + VlhVSjhUeGI4WDlCQXJXaUdGdld2UE0KJGzt/0tQBE4LPKuTh67hi/P3vp+dzH4B + dqzYs8SgwJDMofEDA9b5+FG3tixFnGf6UATz3W2ZQK/WOMchAH5/zw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2025-02-23T03:41:06Z" + mac: ENC[AES256_GCM,data:tM1snUKWYf7CNfg/OwXg/VM334wgqhAfr12Fhka0OiDhFYeXRSTwUcM9U3rByZ9hBdT89q94svioVSxYpc74RGXj+iuX1fVluEdg8KtNSsnw2R351OGQ42ijeeKcGKETTs7zqQMnZM85xPF2tmc2QYj6uPih0UPISYdSzdB561s=,iv:e8RL61ajNASdjrcHnFd3TG1VFgXoj1P/BvwkYSlAfKA=,tag:e767zoOAIwCSYyvWUvABUg==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.9.4 diff --git a/targets/vpn/secrets/secrets.yaml.example b/targets/vpn/secrets/secrets.yaml.example new file mode 100644 index 0000000..8228d48 --- /dev/null +++ b/targets/vpn/secrets/secrets.yaml.example @@ -0,0 +1,2 @@ +age-key: age1zyx... +cloudflare-api-token: d4d... diff --git a/targets/vpn/terraform.tf b/targets/vpn/terraform.tf index 46fa16a..0f04b02 100644 --- a/targets/vpn/terraform.tf +++ b/targets/vpn/terraform.tf @@ -2,6 +2,7 @@ module "vpn" { source = "../../terraform/nixos-vpn" nixos_flake_attr = "vpn" nixos_vars_file = "${path.module}/nixos-vars.json" + sops_file = abspath("${path.module}/secrets/secrets.yaml") tags = { Terraform = "true" Target = "vpn" diff --git a/terraform/nixos-vpn/decrypt-age-keys.sh b/terraform/nixos-vpn/decrypt-age-keys.sh new file mode 100755 index 0000000..7a1df5d --- /dev/null +++ b/terraform/nixos-vpn/decrypt-age-keys.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail -x + +mkdir -p var/lib/secrets + +umask 0177 +sops --extract '["age-key"]' --decrypt "$SOPS_FILE" >./var/lib/secrets/age +# restore umask +umask 0022 diff --git a/terraform/nixos-vpn/main.tf b/terraform/nixos-vpn/main.tf index 29d732e..b5c8a31 100644 --- a/terraform/nixos-vpn/main.tf +++ b/terraform/nixos-vpn/main.tf @@ -28,6 +28,10 @@ module "deploy" { target_host = hcloud_server.nixos_vpn.ipv4_address instance_id = hcloud_server.nixos_vpn.id debug_logging = true + extra_files_script = "${path.module}/decrypt-age-keys.sh" + extra_environment = { + SOPS_FILE = var.sops_file + } } locals { diff --git a/terraform/nixos-vpn/variables.tf b/terraform/nixos-vpn/variables.tf index e2fc775..a8fb8a4 100644 --- a/terraform/nixos-vpn/variables.tf +++ b/terraform/nixos-vpn/variables.tf @@ -21,6 +21,11 @@ variable "nixos_vars_file" { description = "File to write NixOS configuration variables to" } +variable "sops_file" { + type = string + description = "File to SOPS secrets file" +} + variable "nixos_flake_attr" { type = string description = "NixOS configuration flake attribute"