diff --git a/applications/pastebin-stream.nix b/applications/pastebin-stream.nix index af3af13..a60d334 100644 --- a/applications/pastebin-stream.nix +++ b/applications/pastebin-stream.nix @@ -1,7 +1,55 @@ {pkgs, ...}@args: - (import ../lib/node-application.nix) args { - tarball = "https://git.cryto.net/joepie91/pastebin-stream/archive/master.tar.gz"; - name = "pastebin-stream"; - hasErrorReporting = true; - mainBinaryPath = "server.js"; - } + with pkgs.stdenv.lib; + + let + fetchFromCrytoGit = (import ../lib/fetch/from-cryto-git.nix) args; + nodeApplication = (import ../lib/node-application.nix) args; + createJsonConfiguration = (import ../lib/build/json-configuration.nix) args; + in + {errorPath, debugMode ? false, rev, sha256}: + let + configuration = { + errors = { + directory = errorPath; + }; + + scraperSettings = { + pastebinCom = { + listInterval = 60; + listLimit = 100; + pasteInterval = 1; + }; + }; + }; + + configurationFile = createJsonConfiguration { + name = "pastebin-stream-configuration.json"; + contents = configuration; + }; + in + nodeApplication { + name = "pastebin-stream"; + + src = fetchFromCrytoGit { + owner = "joepie91"; + repo = "pastebin-stream"; + inherit rev sha256; + }; + + hasErrorReporting = true; + errorPath = errorPath; + mainBinaryPath = "bin/pastebin-stream"; + setupCommands = '' + cp ${configurationFile} $out/config.json + ''; + + serviceConfig = { + preStart = '' + mkdir -m 0700 -p ${errorPath} + chown pastebin-stream ${errorPath} + ''; + environment = mkIf debugMode { + DEBUG = "pastebinStream:*"; + }; + }; + } diff --git a/lib/build/json-configuration.nix b/lib/build/json-configuration.nix new file mode 100644 index 0000000..d25e547 --- /dev/null +++ b/lib/build/json-configuration.nix @@ -0,0 +1,3 @@ +{pkgs, ...}@args: + {name ? "config.json", contents}: + builtins.toFile name (builtins.toJSON contents) diff --git a/lib/build/node2nix-package.nix b/lib/build/node2nix-package.nix new file mode 100644 index 0000000..ad64401 --- /dev/null +++ b/lib/build/node2nix-package.nix @@ -0,0 +1,12 @@ +{pkgs, ...}@args: + {name, src, setupCommands}: + ((import (pkgs.stdenv.mkDerivation { + name = name; + src = src; + buildCommand = '' + mkdir $out + tar -xzvf $src -C $out + cd $out + ${setupCommands} + ''; + })) {}).package diff --git a/lib/fetch/from-cryto-git.nix b/lib/fetch/from-cryto-git.nix new file mode 100644 index 0000000..0a0e2db --- /dev/null +++ b/lib/fetch/from-cryto-git.nix @@ -0,0 +1,11 @@ +{pkgs, ...}@args: + # FIXME: The below `name` default should be updated to use `gitRepoToName` in 17.09 + {owner, repo, rev, name ? ("${repo}-${rev}-src"), ...}@sourceArgs: + let + baseUrl = "https://git.cryto.net/${owner}/${repo}"; + in + pkgs.fetchurl ({ + inherit name; + url = "${baseUrl}/archive/${rev}.tar.gz"; + meta.homepage = baseUrl; + } // removeAttrs sourceArgs ["owner" "repo" "rev"]) diff --git a/lib/generate/caddy-configuration.nix b/lib/generate/caddy-configuration.nix new file mode 100644 index 0000000..13451af --- /dev/null +++ b/lib/generate/caddy-configuration.nix @@ -0,0 +1,19 @@ +{pkgs, ...}@args: + with pkgs.stdenv.lib; + + applications: + concatStrings (map (application: '' + ${application.hostname} { + timeouts none + + ${optionalString (application.tls == false) "tls off"} + ${optionalString (application?root && application.root != null) "root ${application.root}"} + ${optionalString (application?proxyTarget && application.proxyTarget != null) '' + proxy / ${application.proxyTarget} { + websocket + transparent + } + ''} + ${optionalString (application?config) application.config} + } + '') applications) diff --git a/lib/node-application.nix b/lib/node-application.nix index 3f4db0a..c445c6f 100644 --- a/lib/node-application.nix +++ b/lib/node-application.nix @@ -1,38 +1,39 @@ -{pkgs, config, ...}: {tarball, name, mainBinaryPath, serviceOptions ? {}, serviceConfig ? {}, hasErrorReporting ? false}: +{pkgs, config, ...}@args: {src, name, mainBinaryPath, setupCommands ? "", serviceOptions ? {}, serviceConfig ? {}, hasErrorReporting ? false, errorPath ? null}: with pkgs.stdenv.lib; let - /*serviceName = "node-${name}";*/ - serviceName = "node-foo"; + buildNode2nixPackage = (import ./build/node2nix-package.nix) args; + in let + serviceName = "node-${name}"; cfg = config.services."${serviceName}"; - source = builtins.fetchTarball tarball; - application = (import (pkgs.stdenv.mkDerivation { - src = source; - buildInputs = [ pkgs.node2nix ]; - buildCommand = '' - node2nix -6 --pkg-name nodejs_6_x - ''; - })).package; - errorReporter = (import ./node-error-reporter) { inherit pkgs; }; + + application = buildNode2nixPackage { + name = "${serviceName}-source"; + inherit src setupCommands; + }; + + errorReporter = (import ./node-error-reporter) args; + errorReporterModule = if hasErrorReporting then (errorReporter { + application = application; + applicationName = name; + errorPath = errorPath; + }) else null; in { imports = [ - /*mkIf hasErrorReporting (errorReporter { - application = application; - applicationName = name; - })*/ + errorReporterModule ]; options.services."${serviceName}" = { enable = mkEnableOption "${name}"; } // serviceOptions; - config = mkIf cfg.enable { + config = { # FIXME: What if a username conflict occurs? - users.extraUsers."${name}" = { + users.extraUsers."${name}" = mkIf cfg.enable { description = "${name} Service User"; }; - services."${serviceName}" = { + systemd.services."${serviceName}" = mkIf cfg.enable ({ description = "${name} Service"; wantedBy = ["multi-user.target"]; after = ["network.target"]; @@ -40,7 +41,8 @@ serviceConfig = { ExecStart = "${application}/${mainBinaryPath}"; User = name; + PermissionsStartOnly = true; }; - } // serviceConfig; + } // serviceConfig); }; } diff --git a/lib/node-error-reporter/default.nix b/lib/node-error-reporter/default.nix index 318cc65..c48e072 100644 --- a/lib/node-error-reporter/default.nix +++ b/lib/node-error-reporter/default.nix @@ -1,102 +1,117 @@ -{pkgs}: {applicationName, application}: +{pkgs, config, lib, ...}@args: {applicationName, application, errorPath}: with pkgs.stdenv.lib; let + createJsonConfiguration = (import ../build/json-configuration.nix) args; + optionalValue = (import ../util/optional-value.nix); + in let serviceName = "node-${applicationName}-error-reporter"; - cfg = config.services."${serviceName}"; - # FIXME: report-errors NPM package! - in { - options.services."${serviceName}" = { - enable = mkEnableOption "${name} Error Reporter"; + cfg = config.services."node-${applicationName}".errorReporting; - stackFilter = mkOption { - description = '' - What modules to filter out of the simplified stacktraces - shown in the e-mail report. This can either be the - string "*" (to filter out every third-party module), or - an array of module names to filter. + configurationFile = createJsonConfiguration { + name = "error-reporter-configuration.json"; + contents = (lib.filterAttrs (key: value: key != "enable") cfg) // { + errorPath = errorPath; - Note that the e-mail will always include a JSON - attachment containing the full stacktrace - this setting - purely affects the e-mail body. - ''; - default = "*"; - type = types.either types.str (types.listOf types.str); - }; - - subjectFormat = mkOption { - description = '' - The format for the subject line of the report e-mail. In - this string, `$type` will be replaced with the error - type/name, and `$message` will be replaced with the - error message. - ''; - default = "UNHANDLED ERROR: $type - $message"; - type = types.str; + # The following is to make sure we don't end up with {hostname: null, user: null}, etc., which makes report-errors incorrectly conclude that we want to use a local SMTP server. + smtp = optionalValue (cfg.smtp.hostname != null) cfg.smtp; }; + }; + in + { + options.services."node-${applicationName}".errorReporting = { + enable = mkEnableOption "${name} Error Reporter"; - metadata = { - from = mkOption { + stackFilter = mkOption { description = '' - The sender address displayed on the e-mail report. + What modules to filter out of the simplified stacktraces + shown in the e-mail report. This can either be the + string "*" (to filter out every third-party module), or + an array of module names to filter. + + Note that the e-mail will always include a JSON + attachment containing the full stacktrace - this setting + purely affects the e-mail body. ''; - type = types.str; + default = "*"; + type = types.either types.str (types.listOf types.str); }; - to = mkOption { + subjectFormat = mkOption { description = '' - The address to e-mail reports to. + The format for the subject line of the report e-mail. In + this string, `$type` will be replaced with the error + type/name, and `$message` will be replaced with the + error message. ''; + default = "UNHANDLED ERROR: $type - $message"; type = types.str; }; - }; - smtp = { - hostname = mkOption { - description = '' - The hostname on which the SMTP server can be - reached. - ''; - default = null; - type = types.nullOr types.str; - }; + metadata = { + from = mkOption { + description = '' + The sender address displayed on the e-mail report. + ''; + type = types.str; + }; - port = mkOption { - description = '' - The port number that the SMTP server is accessible - on. - ''; - default = null; - type = types.nullOr types.str; + to = mkOption { + description = '' + The address to e-mail reports to. + ''; + type = types.str; + }; }; - username = mkOption { - description = '' - Your username for the SMTP server. - ''; - default = null; - type = types.nullOr types.str; - }; + smtp = { + hostname = mkOption { + description = '' + The hostname on which the SMTP server can be + reached. + ''; + default = null; + type = types.nullOr types.str; + }; - password = mkOption { - description = '' - Your password for the SMTP server. - ''; - default = null; - type = types.nullOr types.str; - }; + port = mkOption { + description = '' + The port number that the SMTP server is accessible + on. + ''; + default = null; + type = types.nullOr types.str; + }; + + username = mkOption { + description = '' + Your username for the SMTP server. + ''; + default = null; + type = types.nullOr types.str; + }; + password = mkOption { + description = '' + Your password for the SMTP server. + ''; + default = null; + type = types.nullOr types.str; + }; + + }; }; - }; - config = { - services."${serviceName}" = mkIf cfg.enabled { - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - serviceConfig = { - ExecStart = "${pkgs.nodejs_6_x}/bin/node ${application}/node_modules/report-errors/lib/daemon/index.js"; - User = systemd.services."node-${applicationName}".serviceConfig.User; # MARKER + config = { + systemd.services."${serviceName}" = mkIf cfg.enable { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + ExecStart = "${application}/lib/node_modules/pastebin-stream/node_modules/.bin/report-errors ${configurationFile}"; + + # FIXME: Is the below the ideal approach? + User = config.systemd.services."node-${applicationName}".serviceConfig.User; + }; }; }; - }; - } + } diff --git a/lib/root-ssh.nix b/lib/presets/root-ssh.nix similarity index 82% rename from lib/root-ssh.nix rename to lib/presets/root-ssh.nix index b23a12a..8e9489b 100644 --- a/lib/root-ssh.nix +++ b/lib/presets/root-ssh.nix @@ -6,6 +6,6 @@ }; users.users.root.openssh.authorizedKeys.keys = [ - (builtins.readFile ./joepie91.pub) + (builtins.readFile ../joepie91.pub) ]; } diff --git a/lib/presets/tools.nix b/lib/presets/tools.nix new file mode 100644 index 0000000..fe4d9ab --- /dev/null +++ b/lib/presets/tools.nix @@ -0,0 +1,12 @@ +{pkgs, ...}: + { + environment.systemPackages = with pkgs; [ + wget + curl + htop + iotop + iftop + nload + lsof + ]; + } diff --git a/lib/tools.nix b/lib/tools.nix deleted file mode 100644 index f63eef3..0000000 --- a/lib/tools.nix +++ /dev/null @@ -1,11 +0,0 @@ -pkgs: { - environment.systemPackages = with pkgs; [ - wget - curl - htop - iotop - iftop - nload - lsof - ]; -} diff --git a/lib/util/optional-value.nix b/lib/util/optional-value.nix new file mode 100644 index 0000000..0369939 --- /dev/null +++ b/lib/util/optional-value.nix @@ -0,0 +1,5 @@ +condition: value: + if condition then + value + else + null diff --git a/lib/remove-newlines.nix b/lib/util/remove-newlines.nix similarity index 100% rename from lib/remove-newlines.nix rename to lib/util/remove-newlines.nix diff --git a/networks/default.nix b/networks/default.nix index 305fdba..8047b70 100644 --- a/networks/default.nix +++ b/networks/default.nix @@ -1,44 +1,55 @@ { network.description = "Cryto"; - osmium = { config, lib, pkgs, ... }@args: let - proxiedApplications = [{ - hostname = "pastebin-stream.cryto.net"; - tls = false; - root = "${pkgs.valgrind.doc}/share/doc/valgrind/html"; - config = '' + osmium = { config, lib, pkgs, ... }@args: + let + pastebinStream = (import ../applications/pastebin-stream.nix) args; + generateCaddyConfiguration = (import ../lib/generate/caddy-configuration.nix) args; + in let + proxiedApplications = [{ + hostname = "pastebin-stream-dev.cryto.net"; + tls = true; + proxyTarget = "http://localhost:3000"; + }]; + in + { + imports = [ + (pastebinStream { + errorPath = "/var/lib/pastebin-stream/errors"; + rev = "3b7f6ea4ad663b82e7cfd95ae3c65f1a32f0cb0a"; + sha256 = "0w29rwgkjpd9cl42z0n2fy5is730db3mfsqvjmxa7x65nz34d3wj"; + }) + ]; - ''; - }]; + services.caddy = { + enable = true; + agree = true; + email = "admin@cryto.net"; + config = '' + ${generateCaddyConfiguration proxiedApplications} + ''; + }; - generateCaddyHostConfiguration = applications: - lib.concatStrings (map (application: '' - ${application.hostname} { - ${lib.optionalString (application.tls == false) "tls off"} - ${lib.optionalString (application.root != null) "root ${application.root}"} - ${application.config} - } - '') applications); + services.node-pastebin-stream = { + enable = true; - pastebinStream = (import ../applications/pastebin-stream.nix); - in { - imports = [ - (pastebinStream args) - ]; + errorReporting = { + enable = true; - services.caddy = { - enable = true; - agree = true; - email = "admin@cryto.net"; - config = '' - ${generateCaddyHostConfiguration proxiedApplications} - ''; - }; + metadata = { + from = "ops@cryto.net"; + to = "admin@cryto.net"; + }; + }; + }; - networking.firewall.allowedTCPPorts = [ 2015 ]; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; - environment.systemPackages = with pkgs; [ - htop - ]; - }; + environment.systemPackages = with pkgs; [ + htop + ]; + }; } diff --git a/systems/osmium-testing.nix b/systems/osmium-testing.nix index 0615ac2..b91e79c 100644 --- a/systems/osmium-testing.nix +++ b/systems/osmium-testing.nix @@ -1,12 +1,16 @@ let - removeNewlines = (import ../lib/remove-newlines.nix); + removeNewlines = (import ../lib/util/remove-newlines.nix); + presetRootSsh = (import ../lib/presets/root-ssh.nix); in { resources.sshKeyPairs.ssh-key = {}; - osmium = { config, pkgs, ... }: { - deployment.targetEnv = "digitalOcean"; - deployment.digitalOcean.region = "ams2"; - deployment.digitalOcean.size = "512mb"; - #deployment.digitalOcean.authToken = removeNewlines (builtins.readFile ../credentials/digitalocean-auth-token); - } // (import ../lib/root-ssh.nix) // ((import ../lib/tools.nix) pkgs); + osmium = {config, pkgs, ...}@args: let + presetTools = (import ../lib/presets/tools.nix) args; + in + { + deployment.targetEnv = "digitalOcean"; + deployment.digitalOcean.region = "ams2"; + deployment.digitalOcean.size = "512mb"; + #deployment.digitalOcean.authToken = removeNewlines (builtins.readFile ../credentials/digitalocean-auth-token); + } // presetRootSsh // presetTools; }