Merge branch 'rework-monitoring'

.gitattributes

@@ -1 +1 @@
-secrets/** diff=sopsdiffer
+secrets/**/*.y?ml diff=sopsdiffer

.sops.yaml

@@ -9,6 +9,7 @@ keys:
   - &shirley age14ysl953378r2vvy7ft3gwce9xp83pr6wypf5lgx2yjwx2lxra5qs6j8eqe
   - &goldberg age1w3wqxt5t00hjv43dcxlr5rjec5mvuzz9ajc8k04azq0gfx0ncgysu6mdmm
   - &hamilton age1uw83n25fx9th2q5y2yedeyzmtzk5yjtwx0kh054v5r2mxc0utuwqacdf77
+  - &hopper age1hzg5camzwyaj0t89xwu7zr506tk02c2z6k0ayh8pfml2lfvl6assyw2xkk
 creation_rules:
   - path_regex: secrets\/all\/*
     key_groups:
@@ -17,6 +18,7 @@ creation_rules:
         - *shirley
         - *goldberg
         - *hamilton
+        - *hopper
   - path_regex: secrets\/shirley\/*
     key_groups:
       - pgp: [ *e1mo, *adb, *momme ]
@@ -32,3 +34,8 @@ creation_rules:
       - pgp: [ *e1mo, *adb, *momme ]
         age:
         - *hamilton
+  - path_regex: secrets\/hopper\/*
+    key_groups:
+      - pgp: [ *e1mo, *adb, *momme ]
+        age:
+        - *hopper

common/default.nix

@@ -1,22 +1,25 @@
 { config, lib, pkgs, inputs, ... }: {
   imports = [
     ./users.nix
-    ../modules/deployment.nix
+    ../modules/chaosjetzt.nix
     # Monitoring is applicable to all hosts, thus placing it here
-    ../services/monitoring
+    ../services/monitoring/client
   ];
 
-  environment.systemPackages = with pkgs; [
-    htop
-    vim
-    tmux
-    rsync
-    curl
-    wget
-    bat
-    fd
-    ripgrep
-  ];
+  environment = {
+    systemPackages = with pkgs; [
+      htop
+      vim
+      tmux
+      rsync
+      curl
+      wget
+      bat
+      fd
+      ripgrep
+    ];
+    enableAllTerminfo = true;
+  };
 
   nix = {
     package = pkgs.nixVersions.stable;
@@ -55,10 +58,6 @@
     };
   };
   # That way we can't forget to disable the access logs for each individual website
-  services.nginx.appendHttpConfig = ''
-    access_log off;
-    log_not_found off;
-  '';
   security.acme = {
     acceptTerms = true;
     defaults.email = "acme+${config.networking.hostName}@chaos.jetzt";

flake.nix

@@ -54,6 +54,12 @@
           ./hosts/goldberg/configuration.nix
         ];
       };
+      hopper = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+        modules = defaultModules ++ [
+          ./hosts/hopper/configuration.nix
+        ];
+      };
     };
 
     colmena = {

hosts/goldberg/configuration.nix

@@ -1,5 +1,8 @@
-{ lib, pkgs, config, ... }: {
-  cj.deployment.environment = "dev";
+{ lib, config, ... }: {
+  cj = {
+    deployment.environment = "dev";
+    monitoring.interface = "ens10";
+  };
 
   imports = [
     ./hardware-config.nix
@@ -12,6 +15,7 @@
     ../../services/hedgedoc.nix
     ../../services/pretix.nix
     ../../services/pretalx.nix
+    ../../services/monitoring/server
   ];
 
   system.stateVersion = "23.05";
@@ -36,6 +40,9 @@
     environmentFile = lib.mkForce null;
   };
 
+  # Just so the disk won't fill up from the logs
+  services.prometheus.retentionTime = lib.mkForce "5d";
+
   # This is specific to every host!
   systemd.mounts = [{
     what = "/dev/disk/by-id/scsi-0HC_Volume_27793580";

hosts/hamilton/configuration.nix

@@ -1,5 +1,8 @@
-{ pkgs, baseDomain, config, ... }: {
-  cj.deployment.environment = "prod";
+{ config, ... }: {
+  cj = {
+    deployment.environment = "prod";
+    monitoring.interface = "enp7s0";
+  };
 
   imports = [
     ./hardware-config.nix

hosts/hopper/configuration.nix

@@ -0,0 +1,28 @@
+{ ... }: {
+  cj = {
+    deployment.environment = "prod";
+    monitoring.interface = "enp7s0";
+  };
+
+  imports = [
+    ./hardware-config.nix
+    ../../services/monitoring/server
+  ];
+
+  system.stateVersion = "24.11";
+
+  networking = {
+    hostName = "hopper";
+    # Fallback / for the monitoring v(x)lan
+    useDHCP = true;
+    defaultGateway = { address = "172.31.1.1"; interface = "enp1s0"; };
+    defaultGateway6 = { address = "fe80::1"; interface = "enp1s0"; };
+    nameservers = [ "213.133.98.98" "213.133.99.99" "213.133.100.100" ];
+
+    interfaces.enp1s0 = {
+      useDHCP = false;
+      ipv4.addresses = [ { address = "159.69.87.229"; prefixLength = 32; } ];
+      ipv6.addresses = [ { address = "2a01:4f8:c2c:7197::1"; prefixLength = 64; } ];
+    };
+  };
+}

hosts/hopper/hardware-config.nix

@@ -0,0 +1,35 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "virtio_pci" "virtio_scsi" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/9db44501-587a-4862-8eee-76e660bd8aa2";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp7s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/sda";
+}

hosts/shirley/configuration.nix

@@ -1,5 +1,8 @@
-{ pkgs, config, ... }: {
-  cj.deployment.environment = "prod";
+{ ... }: {
+  cj = {
+    deployment.environment = "prod";
+    monitoring.interface = "ens10";
+  };
 
   imports = [
     ./hardware-config.nix

modules/chaosjetzt.nix

@@ -0,0 +1,62 @@
+{ config
+, lib
+, ... }:
+
+let
+  inherit (lib) mkOption types optionalString;
+
+  cfg = config.cj.deployment;
+  isDev = cfg.environment == "dev";
+in
+{
+  options.cj.deployment = {
+    environment = mkOption {
+      description = "Environment this host will be used for. Affects both colmena deploy groups and the baseDomain";
+      type = types.enum [ "dev" "prod" ];
+    };
+  };
+
+  options.cj.monitoring = {
+    interface = mkOption {
+      description = "Interface the monitoring network is attached";
+      type = types.str;
+    };
+
+    blackbox = {
+      "http" = mkOption {
+        type = with types; listOf str;
+        default = [];
+      };
+
+      "tcp_tls" = mkOption {
+        type = with types; listOf str;
+        default = [];
+      };
+    };
+
+    pretix = mkOption {
+      description = "Prometheus endpoints to scrape";
+      type = with types; listOf str;
+      default = [];
+    };
+
+    synapse = mkOption {
+      description = "Port where the metrics listener is located";
+      type = with types; listOf int;
+      default = [];
+    };
+
+    ports = mkOption {
+      description = "List of ports to allow on the monitoring interface (convenience function)";
+      type = with types; listOf port;
+      default = [];
+    };
+  };
+
+  config = {
+    _module.args = {
+      inherit isDev;
+      baseDomain = "${optionalString isDev "dev."}chaos.jetzt";
+    };
+  };
+}

secrets/_admin_gpg_keys/adb_B1480CFF9BBE8E2648A26A640B2E7C171E3AD6D7.asc

@@ -0,0 +1,75 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF9wwHwBEAC+RM6O9wmnQsDlDB+Vr1gYaw54++NX0ORIiTxUOAqcbrd3BfMs
+04VHNVVu2ltVvFjuzK5KX/kGYapsIS7exXxFCATYHhHB9AAw9qRXbKW6GYtx/W8w
+FlEMvl3X6HZb+151pMslIAWluTTfTB8trYmt/f7RPFI4is3NJK31b8haadDQxKtC
+mQprfdnNQbK9Ayv1to2zN1THS3ukaE0a//QwXXu0niqklGWrM7PkOxIC+zBrc7s/
+We4i7O7/J6By1v42OM/chOsqwnk3lfayxyN1A2SxhzVFwVxZUVfvDulnwqPYV1KA
+u5bh5cJLBpciGaX4CAlNX9nrbjvTQ2+vA/DkXyjx3NWEX+3yRU6lK/xgR3y5eUKT
+nU1VEbVgR7pwvhd5eAsldx+DoYW9XIA46HTM7iH8BEheZTNJ4CWx+j/xfHv0zA1+
+VAl3WoBcxeUJ5po0zTY/KYCSK0r/3oa4wrBv2E5ogMJex2CEUWLK1ue74PeE7IBj
++CCalzPcmIiRJQH9mVy9Czlr6uIrgEO94N0NHmCSrPXiOvkzsaIMfmw2Sy5O/ZMn
+ccZaichcaSzs2zIQBvbKd38taj03LJqDXXjB7EzUJcDewy+ErDanfgREM7Z6imHv
+Y820a8ncAq9oVfagRvUIJ6daEZ8PrIHsqwCh8W8mN2wgBJV9IMZmYgrm1QARAQAB
+tCBBbGJhbiBEYXZpZCBCZWNrZXIgPG1haWxAYWRiLnNoPokCNQQQAQgAHwUCX3DA
+fAYLCQcIAwIEFQgKAgMWAgECGQECGwMCHgEACgkQCy58Fx461tdsgBAAoPvYafBh
+MgDQYyenL/l8+wGXCouBgd+N3oIuD9I81SzLkxfeXjIOzk4VOPuFQqrcguWO+ix/
+I73xe/+fX2yhSfmJKOoFgspmfbsLs5lNs4uvezV0PWD4EJYCF4eNz+aX84tQP1oK
+N5waWM9vw1+M2daYbZL2S6P3JPu4wADDsvuwzBsNXhYUBoR2mTBOCTTvriZ2ZasW
+DHlSaRtyCCYm/6LRg+hUtcPDgxZO3hZN4J31Z1TTJlYyeSAtxg+c12TRtiz0bQj2
+m38/atpcPMjXpqq5C1TNHN9iVaUAtjSEHUApZ4bodd7V6ofYj4LYU61/Y2QqEodi
+GV2BNKToDYdun6DMAWaM8T8IGzr7wkiV1sj3dDtxLE0jpI000x7ziexwPuh2EBsT
+u9wgJy88KbIqLTi2DQPmLdYfpUZOBtMKmoHdY8eC0G2TJVDvWTVk+BclQiBCMHe/
+Is3bEWCrLt/Y19yyNHAHngVPU3f9H7ZiOsNvctT8tRG88H9j54QhrsQrjZAvzG06
+j4GsyihCp5TYjuMjYw8oFNsuyN39HLpACSIhU5rIX8uEDgwbeoMaWuYd0K6zXK9j
+6VEphycYYRn7C01tNSiTs1IkX7pJXUVd+r5ga9ccEMRk4pg890YsYvoBp4siNLfl
+qucgFyEVC00tDvN09UzcmC9sPDPdZ1cyl/m0H0FsYmFuIERhdmlkIEJlY2tlciA8
+cmVnQGFkYi5zaD6JAjIEEAEIABwFAl9wwHwGCwkHCAMCBBUICgIDFgIBAhsDAh4B
+AAoJEAsufBceOtbXAKAQAKZZblPxs5LdHqM6nFAM2QSJjnm0ll4I9nKDtwOdCE8k
+Pm/SZgB1JXoSuu3yj0DEmGlg36k7fUMl2hFQJSu5IKTJIscyQwv+C+8ao9Azi27I
+QNg7YLqnLnyKAUQ6xy1tLQHD+BKeZwkLHCc3Egxtm9zje8qOvzS6IWedwXG4J0CW
+UMPy/UqmXMAZA5h0S2VgGJD0fNLIyPGJg9tccfo2mkOC8SKwJ6/8JDlTplIUR3GZ
+y5f54m+ALEPmbhPQQsnFQXCTRvHlX+sgYKtZ4jZfqPp+E/VpPK33VyG9j7IPAQ8/
+s4MERrllpyxZoyHi30nfzCdgI/G6Gp4t9eOx/1tXLSBK0sZ5bN2xVAbbw8uz16Un
+Vr40lQ3sKcmyCGS4ff6CFUH7NR09AiTjPLt8/wwOAXN+R+ZCO4JlcR+LJKlwqKZa
+ukvu7kJ6EX7uryUfoB1gGckX84lSdrwvR3yulFiAKZJczOfeEzs74KWYjcvzBKpE
+J8df+v77FssH+N757ANWFpU/wkak2/reT3ayFLIkoyT+c+G34vCwdTKT7wncqicP
+5SDGDXqdK+HR1jj3/3wOU5RJ9XMns/F6e2jb9oCuI9xBi6/ehDJv5I7L5UMuiebe
+QfehSuzYrNXs+vUx7U5aEkpbODjN4stnuwsU1P2Ze28N6PA6h4Ly1K6vcRLJw8Yh
+tCJBbGJhbiBEYXZpZCBCZWNrZXIgPHNjaG9vbEBhZGIuc2g+iQIyBBABCAAcBQJf
+cMB8BgsJBwgDAgQVCAoCAxYCAQIbAwIeAQAKCRALLnwXHjrW18QNEACX1+AFu/qf
+fsUarMkdPl55ssC0MZ66AkbJX53SQR79bIM8wiM4y6PWwpjnMcfwbXCadEzz5mYi
+bWLJ8a/+Hm0OFjN0IBJAUbMMjZsei46voe+p9p6Wx5e1nSTSgZXPjn0zPFmWj325
+3+VmVAdz331HL7/+Q+WWHai2QAS0y75oqadkY31KTf1pQKjZ/I6XEXCkeURWUAzL
+rOawijwg4zj79fJZgjsifNUSItL/Y38rwNKaOQTEg9TPZSotyV7nl0cbsAIoZEqf
+kfCjSZWpoZCj9/NpEE8PiES32e8XglZ6CL+S3qnwnFMX9sIlc/Z0ZxUYkeC3Z5Q9
+1Aa1ihulFRAhSqYpqE3bcvmTB19WRJ8gwZ4rmvBY21Ae0VJrPe5ggtJ8dHO3ZAL2
+c0rHCF7DwVDQEcWgbxeJmo5YIIwwoz0NXpFTG2vCDSJZTjShnJ1Zq6fXYMBu3WuI
+SBIAEqyhQdP6zU3nb59GZvLDy4z9eudkocYzx1qezQn4yrD1wcH99Q+fD0zwN7YW
+O3tyLZ2tjSHY7gcIAW1gjnBH8AEs4U2CZE/JnFVhuHManerfJZ3A6EkKBZtuqaxL
+gjBaePGT7WJcSvWIG816+SkCEOQoCJ5fLd4SMdpTtegGcbTs1B2n55moLQ19ZURV
+2Qa747a1ifAhBHE6e/XnQGq0OGV5XEImPrkCDQRfcMB8ARAAundX3A1wItju1BAu
+EoeLXx4fnUHLW70hy+K+6sIrC0Wm3gEmu92FSwVL5zkVuXM9/RPLsVll+sop1yzn
+91kpiGe1cm8jWaODJoqGL0REj1BZ++RSgq6YbyHTtgz1/Jnl7OYXBjbXmBROXpjW
+87JSOiil7vPquS/LBaxL9mUvULqyOusGYMy6ld3oL+ZUwb+DJvyeccKvEMztvGXy
+r02zys6DuS+kf4/g40v1dnT08/ybSDKGOIFwYFmwhS2I4omsFnz2zvLReT0Dp9lR
+BN3hhKDgEiPjUroR9XJ9ogXSH+1QPWJd5/zdACcTH3Fxd4S13pA61dM20OG1JNrq
+mGfyuKP5JIYnaYD6cqy7bfJ1M02gCYm3DBjGHOr2vu1kuS8dDhpGtWH4LUYHJx5W
+2UYJGVSubeQn+r9QaWyEthBQZKoWZi/3v6ENH+dcveh/sTq7QCzfZr1jAyNywYvF
+tWBUhcRzLks6gTippWwZuVszzkawNGAEpjGAQ2GdJS0cG813nUK3WEFDqBeM9R1/
+grQ1AWbO+iMHAvXZrLcTv7V8zGHZKAilqeKvHUQFzQDFE7+Uqy/jU58W/U08Gu1h
+xK45ZLQNP3MpW3/D2MJlO7Lr58dPaKAouuiD2kZBdGN8zU70K7+93wcJEgDNc4TQ
+8+c4MxhcZy37ot7Qf2rlVRPOPIMAEQEAAYkCHwQYAQgACQUCX3DAfAIbDAAKCRAL
+LnwXHjrW1ysQD/wOPYn8zMUI2OCwSL1QCe1C9ksx+ZrPidYwKgjQI0DA4qVDX164
+zmQMiAaDPIIMTqt9VK/s9dxyJHNs5YgrlmdcoBSS8B4E6FD22OYZyEXsmBWzpkMl
+4/HJidmsZb3GtGhkxmo30ZC/JQQxpJ0vjxfMzp9KWldVcsdlI65YS0A8TZN4t/wY
+N4xp6iwBzUxCwk670xBtJOAajRkWL5Vy6zhKla2KHUl96JFihg6LLOy/Ph84S+4I
+UKyQbfRhpmMJ8HRnxviwrxDXKeIm0q2sFJ6a8oOdzkDTMRKvAr2vpFf2BqNSJG/1
+iMTIQrPVdLcm8cD3k2Rdi5fccJeYyS4A6keZfdfl5qofY0JBINO2t8uPT6MrR8Fu
+8b/yH2VPWgfNA9rNi3jY2Fkaia8h8Rc59DlNIzPcE0RBHaHx4qGSLG5vsd8/SDuS
+ncA/I6kY5Ky0DLQYDccOvwZq9LoWc53skPp2IhhnOnNVX/aZe51FZH1BfP0UZwBJ
+VhhIBcmX0mKeqiaobrlNH5Ms1lgZfDPwx0cr8di4tJZz+IcRL+3SL6Lrzc4nr9k+
+sIpdsfSulXm1KO4iGLXV603v3qu/UFGHQOikqeHdOARwibyeY77c0hMcySSCduR+
+15sae6m/XnttG4uf2BFKgYEBZVh4AlvL0WSz7GI0adZSVsGKQoyPLvE2yw==
+=5LTY
+-----END PGP PUBLIC KEY BLOCK-----