let
	nixpkgsOptions = {
		overlays = [
			(self: super: {
				/* NOTE: Namespaced under `pkgs.cryto.*` to prevent naming conflicts with upstream nixpkgs */
				cryto = {
					fetchFromCrytoGit = self.callPackage ./lib/fetch/from-cryto-git.nix {};
					nodeApplication = self.callPackage ./lib/node-application.nix {};
					unpack = self.callPackage ./lib/unpack.nix {};
					mobileProxy = self.callPackage ./packages/mobile-proxy { configFile = null; };
					matrixRooms = self.callPackage ./packages/matrix-rooms {};
					pastebinStream = self.callPackage ./packages/pastebin-stream { errorPath = null; };
				};
			})
		];
	};
	pkgs = (import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/nixos-21.11.tar.gz") nixpkgsOptions);
	presets = {
		base = (import ./presets/base.nix);
		kvm = (import ./presets/kvm.nix);
	};
	nginxPresets = {
		phpDisabled = (import ./presets/nginx/php-disabled.nix);
		reverseProxy = (import ./presets/nginx/reverse-proxy.nix);
		letsEncrypt = (import ./presets/nginx/lets-encrypt.nix);
	};
	nodes = (import ./data/nodes.nix);
	tincConfiguration = (import ./lib/tinc-configuration.nix);
	trackSystemMetrics = (import ./lib/track-system-metrics.nix);
	trackServiceMetrics = (import ./lib/track-service-metrics.nix);
	httpHealthChecks = (import ./lib/http-health-checks.nix);
	nginx = (import ./lib/nginx.nix);
	daemon = (import ./lib/daemon.nix);
	errorReporter = (import ./lib/error-reporter.nix);
in {
	network = {
		inherit pkgs;
		description = "Cryto";
	};

	"machine-borg2-01.cryto.net" = let
		self = nodes."machine-borg2-01.cryto.net";
	in { pkgs, lib, ... }: {
		system.stateVersion = "18.09";
		networking.hostName = "machine-borg2-01";

		# FIXME: Why is this needed?
		nixpkgs.overlays = [];

		imports = [
			presets.base
			presets.kvm
			./hardware-configurations/machine-borg2-01.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics self.internalIpv4)
			(trackServiceMetrics self.internalIpv4)
		];

		boot.loader.grub.device = lib.mkForce "/dev/vda";

		users.extraUsers = {
			backup-f0x = {
				createHome = true;
				home = "/home/backup-f0x";
			};
			backup-haless = {
				createHome = true;
				home = "/home/backup-haless";
			};
		};

		users.extraGroups = {
			backup-f0x = { members = [ "backup-f0x" ]; };
			backup-haless = { members = [ "backup-haless" ]; };
		};

		services.borgbackup.repos = {
			"f0x" = {
				allowSubRepos = true;
				quota = "400G";
				path = "/home/backup-f0x";
				user = "backup-f0x";
				group = "backup-f0x";
				authorizedKeys = [
					"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINjJDP2TDyj1X/L6gNgHCXASIWoW/VnJ77FQy39VRTi8 f0x@elephantus"
					"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB+dwsWe1/ujR0N4IxPv7mfyiuKWURc6QwYNJ+VV8KA6 f0x@behemoth"
				];
				authorizedKeysAppendOnly = [
					"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG7WSUY6Y2lsIawo8dPBu4/Omx6c7/1SMD9ve/vpcorN borg-backup@terra"
					"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDeMWPR38zXAbURVTJs+yGDnld5kO7bcgp/70l4wJG0k borg-backup@luna"
				];
			};
			"haless" = {
				allowSubRepos = true;
				path = "/home/backup-haless";
				user = "backup-haless";
				group = "backup-haless";
				authorizedKeys = [
					"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzV5dI01NhwuL6ayiO0STcSQiDf7lEtu63NuLZKQUdZVuVHIqyt3Gquks2OI1NZGrJdXA315yw89ZqyMo+z7gSGHEV6P0fAXKW6G78JOFWsA5lGpaLxTsZ6Q7r0Z9FMqDvA5Jlsyznyj9hhO1cz01WPLzB92ypd9ifldtrAQIYQItxGXOuRkBJiShuIRqtr4Q2chXiOoRZKb4v4Gyt/UPxTpvfM/zcOz0zi1d4ijSbLqgIUJhxvrWADfdgEQ77unepDoD+HT51QBX7dj8RuYivxLSA3vpfNeCgt2CYBf6FYnmWkWSnN1RCtQPJNxsMuLzC2ZBbIkz0tDgcIBPbHxGr sven@linux-rfa7.site"
				];
				authorizedKeysAppendOnly = [
					"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAOpXsDxE7SXeSw/kjgzdwEkNsL9REMabMqYVPM9rem root@machine-haless-03.cryto.net"
				];
			};
		};
	};

	"machine-haless-03.cryto.net" = let
		self = nodes."machine-haless-03.cryto.net";
	in { pkgs, lib, config, ... }@args: {
		system.stateVersion = "19.03";
		networking.hostName = "machine-haless-03";

		imports = [
			presets.base
			presets.kvm
			./hardware-configurations/machine-haless-03.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics self.internalIpv4)
			(trackServiceMetrics self.internalIpv4)
			(httpHealthChecks {
				both = [
					"iomfats.cryto.net"
					"castleroland.cryto.net"
					"awesomedude.cryto.net"
					"matrix-rooms.cryto.net"
					"validatem.cryto.net"
					"nixos-manual-mdx.cryto.net"
					"geojson.cryto.net"
					"ossworks.nl"
				];
			})
			(daemon {
				name = "mobile-proxy";
				displayName = "Mobile Proxy";
				fakeHome = true; # Needed for Babel
				binaryPath = "${pkgs.cryto.mobileProxy.override { configFile = ./data/mobile-proxy/config.jsx; }}/bin/mobile-proxy";
				environmentVariables = {};
			})
			(daemon {
				name = "matrix-rooms";
				displayName = "Matrix Room List Viewer";
				fakeHome = true; # Needed for Babel
				binaryPath = "${pkgs.cryto.matrixRooms}/bin/matrix-room-list-viewer";
				environmentVariables = { NODE_ENV = "production"; };
			})
			(nginx {
				"modular-matrix.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{ root = ./sources/modular-matrix; }
				];
				"geojson.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{ root = ../../image-to-geojson/static; }
				];
				"validatem.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{ root = ../../validatem/site/build; }
				];
				"ossworks.nl" = [
					(nginxPresets.letsEncrypt)
					{ root = ../../ossworks-site/build; }
				];
				"nixos-manual-mdx.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{ root = ../../nixos-manual-mdx/build; }
				];
				"haless.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{
						locations."/shadow/" = {
							alias = ./sources/shadow-generator;
						};
						locations."/knex-mirror/" = {
							alias = ./sources/knex-mirror;
						};
					}
				];
				"books.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.phpDisabled)
				];
				"todo.cryto.net" = [
					(nginxPresets.phpDisabled)
				];
				"learn.cryto.net" = [
					(nginxPresets.phpDisabled)
				];
				"vps-list.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.phpDisabled)
				];
				"iomfats.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://127.0.0.1:3000/")
				];
				"castleroland.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://127.0.0.1:3000/")
				];
				"awesomedude.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://127.0.0.1:3000/")
				];
				"matrix-rooms.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://127.0.0.1:3842/")
				];
			})
		];

		# FIXME: Verify that this actually works...
		services.borgbackup.jobs.system = {
			paths = "/";
			exclude = [
				"/nix"
				"/boot"
				"/sys"
				"/run"
				"/tmp"
				"/dev"
				"/proc"
			];
			repo = "backup-haless@machine-borg2-01.cryto.net:haless-03";
			encryption = {
				mode = "repokey-blake2";
				passphrase = (import ../private/machine-haless-03.cryto.net/borg-passphrase.nix);
			};
			compression = "auto,zlib";
			startAt = "daily";
		};
	};

	"machine-konjassiem-02.cryto.net" = let
		self = nodes."machine-konjassiem-02.cryto.net";
	in { pkgs, lib, config, ... }@args: {
		system.stateVersion = "20.03";
		networking.hostName = "machine-konjassiem-02";

		imports = [
			presets.base
			presets.kvm
			./hardware-configurations/machine-konjassiem-02.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics self.internalIpv4)
			(trackServiceMetrics self.internalIpv4)
			(httpHealthChecks {
				both = [ "git.cryto.net" ];
			})
			(nginx {
				"git.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://127.0.0.1:3000/")
				];
			})
		];

		services.postgresql = {
			enable = true;
			ensureDatabases = [ "gitea" ];
			ensureUsers = [{
				name = "git";
				ensurePermissions = {
					"DATABASE gitea" = "ALL PRIVILEGES";
				};
			}];
		};

		users.users.git = {
			description = "Gitea Service";
			home = "/var/lib/gitea";
			useDefaultShell = true;
			group = "git";
			isSystemUser = true;
		};

		users.groups.git = {};

		# NOTE: Workaround that removes `setuid` from the disallowed syscall list, because otherwise sendmail/opensmtpd breaks
		# systemd.services.gitea.serviceConfig.SystemCallFilter = lib.mkForce "~@clock @cpu-emulation @debug @keyring @memlock @module @mount @obsolete @raw-io @reboot @resources @swap";
		
		# Temporary workaround to make opensmtpd sendmail work (ref. https://github.com/NixOS/nixpkgs/issues/103446)
		# Can remain enabled
		systemd.services.gitea.serviceConfig.PrivateMounts = lib.mkForce true;
		systemd.services.gitea.serviceConfig.PrivateTmp = lib.mkForce true;
		systemd.services.gitea.serviceConfig.ProtectControlGroups = lib.mkForce true;
		systemd.services.gitea.serviceConfig.ProtectHome = lib.mkForce true;
		systemd.services.gitea.serviceConfig.ProtectSystem = lib.mkForce "full"; # downgraded from "strict"

		# Have to be disabled
		systemd.services.gitea.serviceConfig.LockPersonality = lib.mkForce false;
		systemd.services.gitea.serviceConfig.MemoryDenyWriteExecute = lib.mkForce false;
		systemd.services.gitea.serviceConfig.NoNewPrivileges = lib.mkForce false;
		systemd.services.gitea.serviceConfig.PrivateDevices = lib.mkForce false;
		systemd.services.gitea.serviceConfig.PrivateUsers = lib.mkForce false;
		systemd.services.gitea.serviceConfig.ProtectClock = lib.mkForce false;
		systemd.services.gitea.serviceConfig.ProtectHostname = lib.mkForce false;
		systemd.services.gitea.serviceConfig.ProtectKernelLogs = lib.mkForce false;
		systemd.services.gitea.serviceConfig.ProtectKernelModules = lib.mkForce false;
		systemd.services.gitea.serviceConfig.ProtectKernelTunables = lib.mkForce false;
		systemd.services.gitea.serviceConfig.RestrictAddressFamilies = lib.mkForce [];
		systemd.services.gitea.serviceConfig.RestrictRealtime = lib.mkForce false;
		systemd.services.gitea.serviceConfig.RestrictSUIDSGID = lib.mkForce false;
		systemd.services.gitea.serviceConfig.SystemCallArchitectures = lib.mkForce "";
		systemd.services.gitea.serviceConfig.SystemCallFilter = lib.mkForce [];

		# FIXME: This isn't really a secret... and should probably be a part of the Gitea module instead
		deployment.secrets.gitea-header = {
			source = builtins.toFile "header.tmpl" "<style> * { tab-size: 4; } </style>";
			destination = "/var/lib/gitea/custom/templates/custom/header.tmpl";
			owner = { user = "git"; };
			action = [ "systemctl" "restart" "gitea.service" ];
		};

		# FIXME: Healthcheck for this!
		services.gitea = {
			enable = true;
			user = "git";
			appName = "Cryto Git";
			repositoryRoot = "/var/lib/repositories";
			log.level = "Info";

			domain = "git.cryto.net";
			httpAddress = "localhost";
			httpPort = 3000;
			rootUrl = "https://git.cryto.net/";
			
			database = {
				createDatabase = false;
				type = "postgres";
				socket = "/run/postgresql";
				user = "git";
			};

			settings = {
				server = {
					LOCAL_ROOT_URL = "http://localhost:3000/";
				};

				database = {
					LOG_SQL = false;
				};

				service = {
					ENABLE_CAPTCHA = true;
					REGISTER_EMAIL_CONFIRM = true;
					ENABLE_NOTIFY_MAIL = true;
					ENABLE_USER_HEATMAP = false;
				};

				security = {
					PASSWORD_COMPLEXITY = "off";
					INTERNAL_TOKEN = lib.mkForce "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE2MDU0NzQ1Mzh9.XqS6-ha22VNgtUP_mVkZXCMmst-lO8blFAEpWMSlU5g";
				};

				session = {
					PROVIDER = "file";
				};

				mailer = {
					ENABLED = true;
					MAILER_TYPE = "sendmail";
					FROM = "\"Cryto Git\" <noreply@git.cryto.net>";
					SENDMAIL_PATH = "${pkgs.system-sendmail}/bin/sendmail";
				};
			};
		};

		# FIXME: DKIM/DMARC
		services.opensmtpd = {
			enable = true;
			serverConfiguration = ''
				listen on lo

				action "outbound" relay
				match from local for any action "outbound"
			'';
		};
	};

	"machine-pikachu-02.cryto.net" = let
		self = nodes."machine-pikachu-02.cryto.net";
	in { pkgs, lib, config, ... }@args: {
		system.stateVersion = "19.03";
		networking.hostName = "machine-pikachu-02";

		imports = [
			presets.base
			presets.kvm
			./hardware-configurations/machine-pikachu-02.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics self.internalIpv4)
			(trackServiceMetrics self.internalIpv4)
		];
	};

	"machine-osmium-01.cryto.net" = let
		self = nodes."machine-osmium-01.cryto.net";
		pastebinStreamPackage = pkgs.cryto.pastebinStream.override { errorPath = "/var/lib/pastebin-stream/errors"; };
	in { pkgs, lib, config, ... }@args: {
		system.stateVersion = "16.09";
		networking.hostName = "machine-osmium-01";

		imports = [
			presets.base
			./hardware-configurations/machine-osmium-01.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics self.internalIpv4)
			(trackServiceMetrics self.internalIpv4)
			(httpHealthChecks {
				both = [
					"pastebin-stream.cryto.net"
				];
			})
			(daemon {
				name = "pastebin-stream";
				displayName = "pastebin-stream";
				fakeHome = false;
				binaryPath = "${pastebinStreamPackage}/bin/pastebin-stream";
				environmentVariables = {};
			})
			(errorReporter {
				serviceName = "pastebin-stream";
				binaryPath = "${pastebinStreamPackage}/node_modules/.bin/report-errors";
				errorPath = "/var/lib/pastebin-stream/errors";
				from = "ops@cryto.net";
				to = "admin@cryto.net";
			})
			(nginx {
				"pastebin-stream.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://localhost:3000/")
				];
			})
		];
	};

	"machine-workbot-01.cryto.net" = let
		self = nodes."machine-workbot-01.cryto.net";
	in { pkgs, lib, config, ... }@args: {
		system.stateVersion = "18.03";
		networking.hostName = "machine-workbot-01";

		imports = [
			presets.base
			./hardware-configurations/machine-workbot-01.nix
			(tincConfiguration { hostname = self.hostname; nodes = nodes; })
			(trackSystemMetrics "127.0.0.1")
			(trackServiceMetrics "127.0.0.1")
			(httpHealthChecks {
				both = [
					"hydra.cryto.net"
					"prometheus.cryto.net"
					"metrics.cryto.net"
					# "nix-cache.cryto.net" # Not directory-indexable
				];
			})
			(nginx {
				"hydra.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://localhost:3333/")
				];
				"prometheus.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://localhost:9090/")
				];
				"metrics.cryto.net" = [
					(nginxPresets.letsEncrypt)
					(nginxPresets.reverseProxy "http://localhost:8452/")
				];
				"nix-cache.cryto.net" = [
					(nginxPresets.letsEncrypt)
					{ root = "/var/lib/hydra-builds"; }
				];
			})
		];

		services.postgresql = {
			enable = true;
		};

		services.hydra = {
			enable = true;
			port = 3333;
			hydraURL = "http://hydra.cryto.net/";
			notificationSender = "hydra@cryto.net";
			useSubstitutes = false;
			minimumDiskFree = 20;
			minimumDiskFreeEvaluator = 20;
			buildMachinesFiles = pkgs.lib.mkIf (config.nix.buildMachines == []) [];
			extraConfig = ''
				store_uri = file:///var/lib/hydra-builds?secret-key=/var/lib/hydra/binary-cache.key&write-nar-listing=1
				binary_cache_public_uri = https://nix-cache.cryto.net
			'';
		};

		services.prometheus = {
			enable = true;

			globalConfig = {
				scrape_interval = "30s";
			};

			scrapeConfigs = let
				nameInstance = address: name: {
					source_labels = [ "__address__" ];
					target_label = "instance";
					regex = address;
					replacement = name;
				};
				mapToPort = port: builtins.map (host: "${host.internalIpv4}:${builtins.toString port}");
				mapToPortRelabel = port: builtins.map (host: (nameInstance "${host.internalIpv4}:${builtins.toString port}" host.friendlyName));
				# Replace the workbot node (ie. ourselves) with an entry that points directly at localhost instead
				nodes_ = builtins.attrValues (nodes // {
					"machine-workbot-01.cryto.net" = {
						friendlyName = "workbot";
						internalIpv4 = "localhost";
					};
				});
			in [
				{
					job_name = "prometheus";
					static_configs = [{
						targets = [
							"localhost:9090"
						];
					}];
					relabel_configs = [
						(nameInstance "localhost:9090" "workbot")
					];
				} {
					job_name = "nodes";
					scrape_interval = "10s";
					static_configs = [{
						targets = mapToPort 9100 nodes_;
					}];
					relabel_configs = mapToPortRelabel 9100 nodes_;
				} {
					job_name = "systemd";
					scrape_interval = "60s";
					static_configs = [{
						targets = mapToPort 9333 nodes_;
					}];
					relabel_configs = mapToPortRelabel 9333 nodes_;
				}
			];
		};

		services.grafana = {
			enable = true;
			port = 8452;
			rootUrl = "https://metrics.cryto.net/";

			security = let
				credentials = import ../private/grafana-credentials.nix;
			in {
				adminUser = credentials.username;
				adminPassword = credentials.password;
			};

			auth = {
				anonymous.enable = true;
			};
		};

		networking.firewall.allowedTCPPorts = [
			80
			443
		];
	};
}