From 9c31b491d5ec7de733319a568c95e727197e47e0 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Sat, 16 Jun 2018 22:21:56 +0200 Subject: [PATCH] Monster commit; work on Node.js rewrite so far --- .gitignore | 10 +- app.js | 52 ++ frontend/components/task-list.riux | 35 ++ frontend/components/task-list/component.tag | 13 + frontend/components/task.riux | 8 + frontend/index.js | 1 + gulpfile.js | 132 +++++ knexfile.js | 17 + lib/build/wait-for-server.js | 57 +++ lib/disk-store.js | 36 ++ lib/errors.js | 16 + lib/find-and-splice.js | 9 + lib/form-validators/disk-images/add.js | 21 + lib/image-store.js | 60 +++ lib/tag-message.js | 11 + lib/tag-websocket.js | 11 + lib/tasks/fake-task.js | 21 + lib/tasks/progress-indicator.js | 25 + lib/tasks/tracker.js | 107 +++++ lib/validators/one-of.js | 7 + lib/vm/kvm/api.js | 108 +++++ lib/vm/kvm/index.js | 17 + lib/vm/kvm/qmp.js | 94 ++++ lib/websockets/deduplicator.js | 43 ++ lib/websockets/tracker.js | 71 +++ migrations/20160906073423_initial.js | 51 ++ migrations/20160907033335_instance-uuid.js | 13 + migrations/20160907034042_disks.js | 14 + notes/notes.txt | 506 ++++++++++++++++++++ notes/qmp-cdrom-mounted.json | 96 ++++ notes/qmp-cdrom-unmounted.js | 67 +++ notes/qmp-commands.json | 273 +++++++++++ package.json | 67 +++ public/bundle.js | 50 ++ public/css/style.css | 36 ++ routes/disk-images.js | 70 +++ routes/index.js | 9 + routes/instances.js | 27 ++ routes/tasks.js | 112 +++++ scss/style.scss | 65 +++ views/disk-images/add.pug | 40 ++ views/disk-images/list.pug | 19 + views/error.pug | 6 + views/index.pug | 4 + views/instances/list.pug | 25 + views/layout.pug | 17 + 46 files changed, 2542 insertions(+), 7 deletions(-) create mode 100644 app.js create mode 100644 frontend/components/task-list.riux create mode 100644 frontend/components/task-list/component.tag create mode 100644 frontend/components/task.riux create mode 100644 frontend/index.js create mode 100644 gulpfile.js create mode 100644 knexfile.js create mode 100644 lib/build/wait-for-server.js create mode 100644 lib/disk-store.js create mode 100644 lib/errors.js create mode 100644 lib/find-and-splice.js create mode 100644 lib/form-validators/disk-images/add.js create mode 100644 lib/image-store.js create mode 100644 lib/tag-message.js create mode 100644 lib/tag-websocket.js create mode 100644 lib/tasks/fake-task.js create mode 100644 lib/tasks/progress-indicator.js create mode 100644 lib/tasks/tracker.js create mode 100644 lib/validators/one-of.js create mode 100644 lib/vm/kvm/api.js create mode 100644 lib/vm/kvm/index.js create mode 100644 lib/vm/kvm/qmp.js create mode 100644 lib/websockets/deduplicator.js create mode 100644 lib/websockets/tracker.js create mode 100644 migrations/20160906073423_initial.js create mode 100644 migrations/20160907033335_instance-uuid.js create mode 100644 migrations/20160907034042_disks.js create mode 100644 notes/notes.txt create mode 100644 notes/qmp-cdrom-mounted.json create mode 100644 notes/qmp-cdrom-unmounted.js create mode 100644 notes/qmp-commands.json create mode 100644 package.json create mode 100644 public/bundle.js create mode 100644 public/css/style.css create mode 100644 routes/disk-images.js create mode 100644 routes/index.js create mode 100644 routes/instances.js create mode 100644 routes/tasks.js create mode 100644 scss/style.scss create mode 100644 views/disk-images/add.pug create mode 100644 views/disk-images/list.pug create mode 100644 views/error.pug create mode 100644 views/index.pug create mode 100644 views/instances/list.pug create mode 100644 views/layout.pug diff --git a/.gitignore b/.gitignore index bd928f0..3853f5c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ -installer/slave_sfx.py -installer/master_sfx.py -*.pyc -testing -.geanyprj -.sass-cache -sasswatch.* +config.json +node_modules +images diff --git a/app.js b/app.js new file mode 100644 index 0000000..2f92873 --- /dev/null +++ b/app.js @@ -0,0 +1,52 @@ +'use strict'; + +const express = require("express"); +const expressWs = require("express-ws"); +const knex = require("knex"); +const path = require("path"); +const bodyParser = require("body-parser"); + +let db = knex(require("./knexfile")); +let imageStore = require("./lib/image-store")(path.join(__dirname, "./images")); +let taskTracker = require("./lib/tasks/tracker")(); + +let state = {db, imageStore, taskTracker}; + +let app = express(); +expressWs(app); + +app.set("view engine", "pug"); +app.set("views", path.join(__dirname, "views")); + +app.use((req, res, next) => { + res.locals.isUnderPrefix = function isUnderPrefix(path, resultingClass) { + // FIXME: Proper path segment parsing... + if (req.originalUrl.indexOf(path) === 0) { + return resultingClass; + } else { + return ""; + } + } + + next(); +}); + +app.use(express.static(path.join(__dirname, "public"))); + +app.use(bodyParser.urlencoded({ + extended: true +})); + +app.use(require("./routes/index")); +app.use("/disk-images", require("./routes/disk-images")(state)); +app.use("/instances", require("./routes/instances")(state)); + +app.use((err, req, res, next) => { + res.render("error", { + error: err + }); +}); + +app.listen(3000).on("listening", () => { + console.log("Listening..."); +}); diff --git a/frontend/components/task-list.riux b/frontend/components/task-list.riux new file mode 100644 index 0000000..78cbe7e --- /dev/null +++ b/frontend/components/task-list.riux @@ -0,0 +1,35 @@ +task-list + use(path="./task", as="task") + + h1 Task List + + rx-each(for="task", in="{data.tasks}") + task(data="{task}", parent="{tag}", on-click="{logTask(task)}") + + style(type="text/scss"). + .rx-scope { + border: 1px solid black; + + h1 { + color: red; + } + + task { + background-color: rgb(204, 204, 204); + border-bottom: 1px solid rgb(50, 50, 50); + } + + button.rx-all { + font-size: 18px; + font-weight: bold; + } + } + + script. + console.log("data:", data); + console.log("tag:", tag); + + function logTask(tag, task) { + console.log("task:", task); + console.log("this:", tag); + } diff --git a/frontend/components/task-list/component.tag b/frontend/components/task-list/component.tag new file mode 100644 index 0000000..0b65d29 --- /dev/null +++ b/frontend/components/task-list/component.tag @@ -0,0 +1,13 @@ +task-list + task(each="{task in tasks}", task-data="{task}") + + style(type="scss"). + foo { + bar { + asddffasdf: "foo" + } + } + + script. + // fooas + foo.style("foo") diff --git a/frontend/components/task.riux b/frontend/components/task.riux new file mode 100644 index 0000000..59265b5 --- /dev/null +++ b/frontend/components/task.riux @@ -0,0 +1,8 @@ +task + .name { attributes.task.name } + .progress(style="width: {progress(attributes.task)}%;") + + script. + function progress(task) { + return (task.progress / task.total); + } diff --git a/frontend/index.js b/frontend/index.js new file mode 100644 index 0000000..ad9a93a --- /dev/null +++ b/frontend/index.js @@ -0,0 +1 @@ +'use strict'; diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..58d78bc --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,132 @@ +'use strict'; + +const Promise = require("bluebird"); +const path = require("path"); +const gulp = require("gulp"); +const webpackStream = require("webpack-stream"); +const webpack = require("webpack"); +const gulpCached = require("gulp-cached"); +const gulpRename = require("gulp-rename"); +const gulpNodemon = require("gulp-nodemon"); +const gulpLivereload = require("gulp-livereload"); +const gulpNamedLog = require("gulp-named-log"); + +const presetSCSS = require("@joepie91/gulp-preset-scss"); +const partialPatchLivereload = require("@joepie91/gulp-partial-patch-livereload-logger"); + +const serverWaiter = require("./lib/build/wait-for-server")(3000); + +const nodemonLogger = gulpNamedLog("nodemon"); +const livereloadLogger = gulpNamedLog("livereload"); + +partialPatchLivereload(gulpLivereload); + +let nodemon; + +let sources = { + scss: ["./scss/**/*.scss"] +} + +/* The following resolves JacksonGariety/gulp-nodemon#33 */ +process.once("SIGINT", function() { + process.exit(0); +}); + +function tryReload(target) { + if (!serverWaiter.isWaiting()) { + if (typeof target === "string") { + gulpLivereload.changed(target); + } else if (target.path != null) { + if (Array.isArray(target.path)) { + gulpLivereload.changed(target.path[0]); + } else { + gulpLivereload.changed(target.path); + } + } + } +} + +function queueFullReload() { + return Promise.try(() => { + nodemonLogger.log("Waiting for server to start listening...") + return serverWaiter.wait(); + }).then(() => { + nodemonLogger.log("Server started!") + tryReload("*"); + }); +} + +gulp.task("scss", () => { + return gulp.src(sources.scss) + .pipe(presetSCSS({ + basePath: __dirname + })) + .pipe(gulp.dest("./public/css")); +}); + +gulp.task("webpack", () => { + return gulp.src("./frontend/index.js") + .pipe(webpackStream({ + watch: true, + module: { + preLoaders: [{ + test: /\.tag$/, + loader: "riotjs-loader", + exclude: /node_modules/, + query: { + type: "babel", + template: "pug", + parserOptions: { + js: { + presets: ["es2015-riot"] + } + } + } + }], + loaders: [ + { test: /\.js$/, loader: "babel-loader" }, + { test: /\.json$/, loader: "json-loader" } + ] + }, + plugins: [ new webpack.ProvidePlugin({riot: "riot"}) ], + resolveLoader: { root: path.join(__dirname, "node_modules") }, + resolve: { + extensions: [ + "", + ".tag", + ".web.js", ".js", + ".web.json", ".json" + ] + }, + debug: false + })) + .pipe(gulpRename("bundle.js")) + .pipe(gulp.dest("./public")); +}); + +gulp.task("nodemon", ["scss"], () => { + nodemon = gulpNodemon({ + script: "./app.js", + delay: 500, + ignore: ["node_modules", "public", "gulpfile.js"], + quiet: true + }).on("restart", (cause) => { + nodemonLogger.log("Restarting... caused by:", cause.join(" / ")); + serverWaiter.reportRestart(); + }).on("start", () => { + return queueFullReload() + }); +}); + +gulp.task("watch-server", ["nodemon"], () => { + gulpLivereload.listen({ + quiet: true + }); + + gulp.watch(sources.scss, ["scss"]); + + gulp.watch("./public/**/*", tryReload); +}); + +gulp.task("watch", ["watch-server", "webpack"]); +gulp.task("default", ["watch"]); diff --git a/knexfile.js b/knexfile.js new file mode 100644 index 0000000..6e44f7e --- /dev/null +++ b/knexfile.js @@ -0,0 +1,17 @@ +'use strict'; + +const config = require("./config.json"); + +module.exports = { + client: "pg", + connection: { + database: "cvm", + charset: "utf8", + username: config.database.username, + password: config.database.password + }, + pool: { + min: 2, + max: 10 + } +} diff --git a/lib/build/wait-for-server.js b/lib/build/wait-for-server.js new file mode 100644 index 0000000..12ef480 --- /dev/null +++ b/lib/build/wait-for-server.js @@ -0,0 +1,57 @@ +'use strict'; + +const net = require("net"); +const Promise = require("bluebird"); + +function listening(port, cb) { + let done = false; + let sock = new net.Socket(); + sock.setTimeout(50); + sock.on("connect", () => { + done = true; + cb(null, true); + sock.destroy(); + }).on("timeout", () => { + if (done) { return; } + cb(null, false); + }).on("error", (err) => { + if (done) { return; } + cb(null, false); + }).connect(port); +} + +function waitForServer(port, cb) { + setTimeout(() => { + listening(port, (err, isListening) => { + if (err != null) { + cb(err); + } else { + if (isListening) { + cb(null, true); + } else { + waitForServer(port, cb); + } + } + }); + }, 100); +} + +module.exports = function serverWaiter(port) { + let isWaiting = false; + + return { + reportRestart: function() { + isWaiting = true; + }, + isWaiting: function() { + return isWaiting; + }, + wait: function() { + return Promise.try(() => { + return Promise.promisify(waitForServer)(port); + }).then((result) => { + isWaiting = false; + }); + } + } +} diff --git a/lib/disk-store.js b/lib/disk-store.js new file mode 100644 index 0000000..76a9da4 --- /dev/null +++ b/lib/disk-store.js @@ -0,0 +1,36 @@ +'use strict'; + +const Promise = require("bluebird"); +const uuid = require("uuid"); +const childProcess = Promise.promisifyAll(require("child-process"), {multiArgs: true}); + +module.exports = function(storePath) { + function getPath(id) { + return path.join(storePath, `${id}.img`); + } + + return { + getPath: function getDisk(id) { + return getPath(id); + }, + create: function createDisk(size, {type} = {type: "qcow2"}) { + return Promise.try(() => { + let diskId = uuid.v4(); + + return Promise.try(() => { + childProcess.execFileAsync("qemu-img", [ + "create", + "-f", type, + getPath(diskId), + size + ]); + }).then(([stdout, stderr]) => { + return diskId; + }); + }); + }, + resize: function resizeDisk(id, size) { + + } + } +} diff --git a/lib/errors.js b/lib/errors.js new file mode 100644 index 0000000..71eb5ea --- /dev/null +++ b/lib/errors.js @@ -0,0 +1,16 @@ +'use strict'; + +const createError = require("create-error"); + +let HttpError = createError("HttpError", { + exposeToUser: true +}); + +module.exports = { + UnauthorizedError: createError(HttpError, "UnauthorizedError", { + statusCode: 401 + }), + ForbiddenError: createError(HttpError, "ForbiddenError", { + statusCode: 403 + }) +} diff --git a/lib/find-and-splice.js b/lib/find-and-splice.js new file mode 100644 index 0000000..cf12d25 --- /dev/null +++ b/lib/find-and-splice.js @@ -0,0 +1,9 @@ +'use strict'; + +module.exports = function findAndSplice(array, object) { + let index = array.indexOf(object); + + if (index !== -1) { + array.splice(index); + } +} diff --git a/lib/form-validators/disk-images/add.js b/lib/form-validators/disk-images/add.js new file mode 100644 index 0000000..b29704c --- /dev/null +++ b/lib/form-validators/disk-images/add.js @@ -0,0 +1,21 @@ +'use strict'; + +const checkit = require("checkit"); +const oneOf = require("../../validators/one-of"); + +module.exports = checkit({ + name: "string", + description: "string", + source: ["required", "string", oneOf([ + "local", + "http" + ])] +}).maybe({ + url: ["required", "string"] +}, (input) => { + return (input.source === "http"); +}).maybe({ + path: ["required", "string"] +}, (input) => { + return (input.source === "local"); +}); diff --git a/lib/image-store.js b/lib/image-store.js new file mode 100644 index 0000000..ebce96c --- /dev/null +++ b/lib/image-store.js @@ -0,0 +1,60 @@ +'use strict'; + +const Promise = require("bluebird"); +const bhttp = require("bhttp"); +const uuid = require("uuid"); +const fs = Promise.promisifyAll(require("fs-extra")); +const path = require("path"); +const endOfStreamAsync = Promise.promisify(require("end-of-stream")); + +const progressIndicator = require("./tasks/progress-indicator"); + +module.exports = function createImageStore(storagePath) { + function getPath(id) { + return path.join(storagePath, `${id}.iso`); + } + + return { + addFromUrl: function addFromUrl(url) { + return Promise.try(() => { + return bhttp.get(url, {stream: true}); + }).then((response) => { + if (response.statusCode !== 200) { + throw new Error(`Encountered a non-200 status code while attempting to download image (status code was ${response.statusCode})`); + } + + let imageId = uuid.v4(); + let targetStream = fs.createWriteStream(getPath(imageId)); + let totalSize = null; + + if (response.headers["content-length"] != null) { + totalSize = parseInt(response.headers["content-length"]); + } + + let downloadTracker = progressIndicator(totalSize, imageId); + + response.on("progress", (completedBytes, totalBytes) => { + downloadTracker.report(completedBytes); + }); + + response.pipe(targetStream); + + // FIXME: Potential race condition? Are writes handled asynchronously? If so, then we may run into issues here, if the downloadTracker completion is based on the last source read, rather than the last destination write. + return downloadTracker; + }); + }, + addFromPath: function addFromPath(path) { + return Promise.try(() => { + let imageId = uuid.v4(); + + return Promise.try(() => { + return fs.copyAsync(path, getPath(imageId), { + overwrite: false + }); + }).then(() => { + return imageId; + }); + }) + } + } +} diff --git a/lib/tag-message.js b/lib/tag-message.js new file mode 100644 index 0000000..cd27428 --- /dev/null +++ b/lib/tag-message.js @@ -0,0 +1,11 @@ +'use strict'; + +const uuid = require("uuid"); + +module.exports = function tagMessage(message) { + if (message.messageId == null) { + message.messageId = uuid.v4(); + } + + return message; +} diff --git a/lib/tag-websocket.js b/lib/tag-websocket.js new file mode 100644 index 0000000..c608702 --- /dev/null +++ b/lib/tag-websocket.js @@ -0,0 +1,11 @@ +'use strict'; + +const uuid = require("uuid"); + +module.exports = function(socket) { + if (socket.socketId == null) { + socket.socketId = uuid.v4(); + } + + return socket; +} diff --git a/lib/tasks/fake-task.js b/lib/tasks/fake-task.js new file mode 100644 index 0000000..966a783 --- /dev/null +++ b/lib/tasks/fake-task.js @@ -0,0 +1,21 @@ +'use strict'; + +const progressIndicator = require("./progress-indicator"); + +let maxProgressValue = 10000; + +module.exports = function createFakeTask(duration) { + let fakeProgressTracker = progressIndicator(maxProgressValue); + let currentProgress = 0; + + function addProgress() { + currentProgress += 1; + fakeProgressTracker.report(currentProgress); + + if (currentProgress < maxProgressValue) { + setTimeout(addProgress, duration / maxProgressValue); + } + } + + return fakeProgressTracker; +}; diff --git a/lib/tasks/progress-indicator.js b/lib/tasks/progress-indicator.js new file mode 100644 index 0000000..412e264 --- /dev/null +++ b/lib/tasks/progress-indicator.js @@ -0,0 +1,25 @@ +'use strict'; + +const Promise = require("bluebird"); +const createEventEmitter = require("create-event-emitter"); + +module.exports = function(total, completionValue) { + return createEventEmitter({ + await: function awaitCompletion() { + return new Promise((resolve, reject) => { + this.once("completed", () => { + resolve(completionValue); + }).once("error", (err) => { + reject(err); + }); + }); + }, + report: function reportStatus(status, description) { + this.emit("progress", status / total, description, status, total); + + if (status >= total) { + this.emit("completed"); + } + } + }); +} diff --git a/lib/tasks/tracker.js b/lib/tasks/tracker.js new file mode 100644 index 0000000..1f6df79 --- /dev/null +++ b/lib/tasks/tracker.js @@ -0,0 +1,107 @@ +'use strict'; + +const assureArray = require("assure-array"); +const createEventEmitter = require("create-event-emitter"); +const uuid = require("uuid"); + +const findAndSplice = require("../find-and-splice"); + +module.exports = function createTaskTracker() { + let tasks = []; + + return createEventEmitter{ + addTask: function addTask(name, indicator, userIds, roleIds, options) { + let mergedOptions = Object.assign({ + removeWhenDone: true + }, options); + + let indicatorType; + + if (typeof indicator.completed === "function" && typeof indicator.finish === "function") { + indicatorType = "are-we-there-yet"; + } else if (typeof indicator.await === "function" && typeof indicator.report === "function") { + indicatorType = "progress-indicator"; + } else { + throw new Error("Unrecognized indicator type"); + } + + let task = createEventEmitter({ + id: uuid.v4(), + name: name, + started: Date.now(), + userIds: assureArray(userIds), + roleIds: assureArray(roleIds), + indicator: indicator, + type: indicatorType, + progress: 0, + lastOperation: null, + lastUpdated: null + }); + + let reportChange = () => { + this.emit("progress", task, task.progress, task.lastOperation); + task.emit("progress", task.progress, task.lastOperation); + } + + let reportCompleted = () => { + this.emit("completed", task); + task.emit("completed"); + + if (mergedOptions.removeWhenDone) { + this.removeTask(task); + } + } + + tasks.push(task); + + if (indicatorType === "are-we-there-yet") { + indicator.on("change", (name, completed, tracker) => { + task.progress = completed; + task.lastOperation = name; + task.lastUpdated = Date.now(); + reportChange(); + + if (completed === 1) { + reportCompleted(); + } + }); + } else if (indicatorType === "progress-indicator") { + indicator.on("progress", (progress, description, completed, total) => { + task.progress = progress; + task.lastOperation = description; + task.lastUpdated = Date.now(); + reportChange(); + }); + + indicator.on("completed", () => { + reportCompleted(); + }); + } + + this.emit("newTask", task); + + assureArray(userIds).forEach((userId) => { + this.emit(`newTask:user:${userId}`, task); + }); + + assureArray(roleIds).forEach((roleId) => { + this.emit(`newTask:role:${roleId}`, task); + }); + + return task; + }, + removeTask: function removeTask(task) { + // FIXME: Do we need to remove event handlers to prevent a memory leak here? + findAndSplice(tasks, task); + }, + byUser: function getTasksByUser(userId) { + return tasks.filter((task) => task.userIds.includes(userId)); + }, + byRole: function getTasksByRole(roleId) { + return tasks.filter((task) => task.roleIds.includes(roleId)) + }, + all: function getAllTasks() { + return tasks; + } + }); +} diff --git a/lib/validators/one-of.js b/lib/validators/one-of.js new file mode 100644 index 0000000..85ff274 --- /dev/null +++ b/lib/validators/one-of.js @@ -0,0 +1,7 @@ +'use strict'; + +module.exports = function createOneOf(validValues) { + return function oneOf(value) { + return (validValues.indexOf(value) !== -1); + } +} diff --git a/lib/vm/kvm/api.js b/lib/vm/kvm/api.js new file mode 100644 index 0000000..75ea51f --- /dev/null +++ b/lib/vm/kvm/api.js @@ -0,0 +1,108 @@ +'use strict'; + +module.exports = function(socket, options) { + return Promise.try(() => { + return Promise.all([ + socket.execute("query-commands"), + socket.execute("query-block") + ]); + }).then((commands, blockDevices) => { + let commandList = commands.map((item) => item.name); + + let supportsBlockdevChangeMedium = commandList.includes("blockdev-change-medium"); + + let cdromDevices = blockDevices.filter((devices) => device.device.match(/^ide[0-9]-cd[0-9]$/)); + + return { + create: function() { + + }, + getSupportedFeatures: function() { + return [ + "start", "stop", "forceStop", "forceReset", + "insertDisk", "ejectDisk", "forceEjectDisk", "inspectDisk", + "vncPassword" + ]; + }, + getStatus: function() { + return socket.execute("query-status"); + }, + start: function() { + // FIXME + }, + stop: function(force = false) { + return Promise.try(() => { + if (force) { + socket.execute("quit"); + } else { + socket.execute("system_powerdown"); + } + }); + }, + reset: function(force = true) { + return Promise.try(() => { + if (force) { + socket.execute("system_reset"); + } else { + throw new NotImplementedError("ACPI reset is not available for QEMU/KVM"); + } + }); + }, + suspend: function() { + return socket.execute("stop"); + }, + unsuspend: function() { + return socket.execute("cont"); + }, + insertDisk: function(path, options = {}) { + return Promise.try(() => { + if (cdromDevices.length === 0) { + throw new Error("No CD-ROM devices available"); + } else { + if (supportsBlockdevChangeMedium) { + return socket.execute("blockdev-change-medium", { + device: cdromDevices[0].device, + filename: path, + format: options.format + }); + } else { + if (options.format != null) { + throw new Error("QEMU/QMP version does not allow for specifying the image format"); + } else { + return socket.execute("change", { + device: cdromDevices[0].device, + target: path + }); + } + } + } + }) + }, + ejectDisk: function(force = false) { + return Promise.try(() => { + if (cdromDevices.length === 0) { + throw new Error("No CD-ROM devices available"); + } else { + return socket.execute("eject", { + device: cdromDevices[0].device + }); + } + }) + }, + inspectDisk: function() { + // FIXME: Normalized format? + return Promise.try(() => { + if (cdromDevices.length === 0) { + throw new Error("No CD-ROM devices available"); + } else { + return Promise.try(() => { + return socket.execute("query-block"); + }).then((blockDevices) => { + return blockDevices.filter((device) => device.device === cdromDevices[0].device); + }); + } + }); + } + } + }); +} diff --git a/lib/vm/kvm/index.js b/lib/vm/kvm/index.js new file mode 100644 index 0000000..d1841f1 --- /dev/null +++ b/lib/vm/kvm/index.js @@ -0,0 +1,17 @@ +'use strict'; + +const path = require("path"); +const createError = require("create-error"); + +const qmp = require("./qmp"); +const api = require("./api"); + +const NotImplementedError = createError("NotImplementedError"); + +module.exports = function createKvmWrapper(instanceId, options) { + return Promise.try(() => { + return qmp(path.join(__dirname, `../../../qmp-sockets/${instanceId}`)); + }).then((socket) => { + return api(socket, Object.assign({instanceId: instanceId}, options)); + }); +} diff --git a/lib/vm/kvm/qmp.js b/lib/vm/kvm/qmp.js new file mode 100644 index 0000000..e2b6434 --- /dev/null +++ b/lib/vm/kvm/qmp.js @@ -0,0 +1,94 @@ +'use strict'; + +const net = require("net"); +const createEventEmitter = require("create-event-emitter"); +const jsonStream = require("JSONStream"); +const createError = require("create-error"); +const defaultValue = require("default-value"); + +const QMPError = createError("QMPError"); +const CommandNotFoundError = createError(QMPError, "CommandNotFoundError"); + +module.exports = function connect(socketPath) { + return Promise.try(() => { + let socket = net.createConnection({path: socketPath}); + let commandQueue = []; + let currentCommand; + + function trySendCommand() { + /* This enforces single-concurrency by only executing a command when there's not already a command being processed. Every time a command is either queued or completed, this function is called again, so that eventually the command queue will drain, executing each command in order. */ + if (currentCommand == null) { + currentCommand = commandQueue.shift(); + socket.write(JSON.stringify(currentCommand.payload)); + } + } + + function commandResult(result) { + if (currentCommand != null) { + currentCommand.resolve(result.return); + } else { + // FIXME: Log a warning! + } + + currentCommand = null; + trySendCommand(); + } + + function commandFailed(result) { + if (currentCommand != null) { + let err; + + if (result.error.class === "CommandNotFound") { + err = new CommandNotFoundError(result.error.desc, result.error); + } else { + err = new QMPError(defaultValue(result.error.desc, "Unknown error occurred"), result.error); + } + + currentCommand.reject(err); + } else { + // FIXME: Log a warning! + } + + currentCommand = null; + trySendCommand(); + } + + let emitter = createEventEmitter({ + execute: function executeCommand(command, args) { + return new Promise((resolve, reject) => { + /* We need to implement a defer here, because the QMP API doesn't tie responses to requests in any way. We can't really do this with .once event listeners either, because 1) that gets messy quickly and is hard to debug when it breaks, and 2) we need a command queue so that we are only ever executing a single command at a time. */ + + commandQueue.push({ + resolve: resolve, + reject: reject, + payload: { + execute: command, + arguments: args + } + }); + + trySendCommand(); + }); + } + }); + + socket.pipe(jsonStream.parse()).on("data", (obj) => { + if (obj.event != null) { + emitter.emit(obj.event, obj); + } else if (obj.error != null) { + commandFailed(obj); + } else if (obj.return != null) { + commandResult(obj); + } else { + throw new Error("Encountered unexpected message type", obj); + } + }); + + return Promise.try(() => { + /* This initializes the QMP API. If it fails, our `connect` Promise will fail as well (as it should). */ + emitter.execute("qmp_capabilities"); + }).then((result) => { + return emitter; + }); + }); +} diff --git a/lib/websockets/deduplicator.js b/lib/websockets/deduplicator.js new file mode 100644 index 0000000..4168fce --- /dev/null +++ b/lib/websockets/deduplicator.js @@ -0,0 +1,43 @@ +'use strict'; + +const events = require("events"); + +module.exports = function createDeduplicator(options) { + let mergedOptions = Object.assign({ + historyLength: 10 + }, options); + + let seenMessages = {}; // FIXME: Use a Set instead? + + function tryMessage(client, data, callback) { + if (client.socketId == null) { + // FIXME: Warning! + console.error("Not deduplicating message because client.socketId is null"); + callback(); + } else if (data.messageId == null) { + // FIXME: Warning! + console.error("Not deduplicating message because data.messageId is null"); + callback(); + } else { + if (seenMessages[client.socketId] == null) { + seenMessages[client.socketId] = []; + } + + let clientSeenMessages = seenMessages[client.socketId]; + + if (clientSeenMessages.indexOf(data.messageId) !== 1) { + clientSeenMessages.push(data); + + if (clientSeenMessages.length > mergedOptions.historyLength) { + clientSeenMessages.shift(); + } + + callback(); + } + } + } + + tryMessage.forgetClient = function forgetClient(client) { + delete seenMessages[client.socketId]; + } +} diff --git a/lib/websockets/tracker.js b/lib/websockets/tracker.js new file mode 100644 index 0000000..786c11f --- /dev/null +++ b/lib/websockets/tracker.js @@ -0,0 +1,71 @@ +'use strict'; + +const websocketDeduplicator = require("./deduplicator"); +const findAndSplice = require("../find-and-splice"); + +module.exports = function(options) { + let deduplicator, externalDeduplicatorProvided; + + if (options.deduplicator != null) { + deduplicator = options.deduplicator; + externalDeduplicatorProvided = true; + } else { + deduplicator = websocketDeduplicator(); + externalDeduplicatorProvided = false; + } + + if (options.namespaced) { + let clients = {}; + + return { + add: function addClient(namespace, client) { + if (clients[namespace] == null) { + clients[namespace] = []; + } + + clients[namespace].push(client); + }, + remove: function removeClient(client) { + Object.keys(clients).forEach((namespace) => { + this.removeFromNamespace(namespace, client); + }); + + if (externalDeduplicatorProvided === false) { + deduplicator.forgetClient(client); + } + }, + removeFromNamespace: function removeClientFromNamespace(namespace, client) { + findAndSplice(clients[namespace], client); + }, + emit: function emit(namespace, data) { + let namespacedClients = clients[namespace]; + + if (namespacedClients != null) { + namespacedClients.forEach((client) => { + deduplicator(client, data, () => { + client.send(JSON.stringify(data)); + }); + }); + } + } + } + } else { + let clients = []; + + return { + add: function addClient(client) { + clients.push(client); + }, + remove: function removeClient(client) { + findAndSplice(clients, client); + }, + emit: function emit(data) { + clients.forEach((client) => { + deduplicator(client, data, () => { + client.send(JSON.stringify(data)); + }); + }); + } + } + } +} diff --git a/migrations/20160906073423_initial.js b/migrations/20160906073423_initial.js new file mode 100644 index 0000000..ee6638b --- /dev/null +++ b/migrations/20160906073423_initial.js @@ -0,0 +1,51 @@ +'use strict'; + +exports.up = function(knex, Promise) { + return Promise.all([ + knex.schema.createTable("images", (table) => { + table.increments("id"); + table.integer("userId").notNullable(); // user that added it + table.uuid("fileId").notNullable(); + table.text("name"); + table.text("description"); + table.enum("sourceType", ["local", "http", "upload"]).notNullable(); + table.text("source"); // URL, path, etc. + table.enum("imageType", ["disk", "tarball"]).notNullable(); // eg. tarballs for OpenVZ + table.boolean("public").notNullable(); // whether the image should be visible to everybody, or just its owner + table.boolean("isInstallMedium").notNullable(); // whether the image is just for installation (if not, it will be directly clonable) + }), + knex.schema.createTable("instances", (table) => { + table.increments("id"); + table.integer("userId").notNullable(); + table.integer("imageId"); + table.integer("lastInstallationMediaId"); + table.text("comment"); + table.text("customIdentifier"); + table.enum("virtualizationType", ["kvm"]).notNullable(); + table.integer("memory").notNullable(); // in MB + table.integer("swap"); // in MB + table.integer("diskSpace").notNullable(); // in MB + table.integer("traffic"); // in MB + table.boolean("suspended").notNullable(); + table.text("suspensionReason"); + table.boolean("terminated").notNullable(); + table.text("terminationReason"); + table.boolean("running"); + }), + knex.schema.createTable("users", (table) => { + table.increments("id"); + table.text("username").notNullable(); + table.text("hash").notNullable(); + table.text("emailAddress").notNullable(); + table.boolean("active").notNullable(); + }) + ]); +}; + +exports.down = function(knex, Promise) { + return Promise.all([ + knex.schema.dropTable("images"), + knex.schema.dropTable("instances"), + knex.schema.dropTable("users") + ]); +}; diff --git a/migrations/20160907033335_instance-uuid.js b/migrations/20160907033335_instance-uuid.js new file mode 100644 index 0000000..13e1805 --- /dev/null +++ b/migrations/20160907033335_instance-uuid.js @@ -0,0 +1,13 @@ +'use strict'; + +exports.up = function(knex, Promise) { + return knex.schema.table("instances", (table) => { + table.uuid("instanceUuid").notNullable(); + }); +}; + +exports.down = function(knex, Promise) { + return knex.schema.table("instances", (table) => { + table.dropColumn("instanceUuid"); + }) +}; diff --git a/migrations/20160907034042_disks.js b/migrations/20160907034042_disks.js new file mode 100644 index 0000000..c32c74c --- /dev/null +++ b/migrations/20160907034042_disks.js @@ -0,0 +1,14 @@ +'use strict'; + +exports.up = function(knex, Promise) { + return knex.table("storage_volumes", (table) => { + table.increments("id"); + table.integer("instanceId"); + table.uuid("volumeUuid").notNullable(); + table.enum("format", ["qcow2"]).notNullable(); + }) +}; + +exports.down = function(knex, Promise) { + +}; diff --git a/notes/notes.txt b/notes/notes.txt new file mode 100644 index 0000000..65c0d93 --- /dev/null +++ b/notes/notes.txt @@ -0,0 +1,506 @@ +qemu-img create -f qcow2 test.qcow 5G +qemu-kvm -cdrom ~/Downloads/debian-8.5.0-amd64-netinst.iso -hda test.qcow -boot d -netdev user,id=user.0 -device e1000,netdev=user.0 -m 196 -localtime + +# Getting Started +https://fedoraproject.org/wiki/How_to_use_qemu +https://en.wikibooks.org/wiki/QEMU/Images +https://nixos.org/wiki/QEMU_guest_with_networking_and_virtfs +https://mbharatkumar.wordpress.com/2010/10/09/qemu-getting-started/ +http://www.nico.schottelius.org/blog/control-and-shutdown-qemu-kvm-vm-via-unix-socket/ +http://man.cx/qemu-system-x86_64(1) + +# Flags +https://wiki.gentoo.org/wiki/QEMU/Options + +# Monitor +https://en.wikibooks.org/wiki/QEMU/Monitor + +# QMP +https://raw.githubusercontent.com/qemu/qemu/master/docs/qmp-intro.txt +https://raw.githubusercontent.com/qemu/qemu/master/docs/qmp-spec.txt +https://raw.githubusercontent.com/qemu/qemu/master/qmp-commands.hx +https://raw.githubusercontent.com/qemu/qemu/master/docs/qmp-events.txt + +http://wiki.qemu.org/QMP +https://kashyapc.com/2013/03/31/multiple-ways-to-access-qemu-machine-protocol-qmp/ +https://www.npmjs.com/package/qemu-qmp + +# QMP (old) +https://qemu.weilnetz.de/w64/2012/2012-06-28/qmp-commands.txt +https://lxr.missinglinkelectronics.com/qemu/qmp-commands.hx + +# Networking +http://www.linux-kvm.org/page/Networking +http://wiki.qemu.org/Documentation/Networking +https://pve.proxmox.com/wiki/Network_Model +http://hyperlogos.org/page/HOWTO-kvm-vde-networking-Ubuntu-Debian-et-cetera + +# Disk Images +https://alexeytorkhov.blogspot.nl/2009/09/mounting-raw-and-qcow2-vm-disk-images.html +https://edoceo.com/cli/qemu +http://libguestfs.org/ +http://www.linux-kvm.org/page/Change_cdrom + +# Storage Pools +https://libvirt.org/storage.html + +# Virtio +http://www.linux-kvm.org/page/Boot_from_virtio_block_device + +# Resource throttling +https://vpsboard.com/topic/4601-kvm-anti-abuse-how-do-you-counter-abuse-with-kvm-users/ - "cgroups cpuacct not a workable solution?" +https://vpsboard.com/topic/4601-kvm-anti-abuse-how-do-you-counter-abuse-with-kvm-users/?do=findComment&comment=66078 - "aka as each vm is a prossess use nice to limit cpu." +http://wiki.qemu.org/Features/AutoconvergeLiveMigration - dynamic CPU throttling lead... +https://lists.gnu.org/archive/html/qemu-devel/2015-06/msg06737.html + +# References +http://www.linux-kvm.org/page/HOWTO +http://www.linux-kvm.org/page/Management_Tools +https://github.com/ChoHag/nbsvm/blob/master/nbsvm +https://github.com/digitalocean/go-qemu/ + +# Full documentation +http://wiki.qemu.org/download/qemu-doc.html + + + +{ + "return": [{ + "io-status": "ok", + "device": "ide1-cd0", + "locked": false, + "removable": true, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 258998272, + "filename": "/home/sven/Downloads/debian-8.5.0-amd64-netinst.iso", + "format": "raw", + "actual-size": 259006464, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": true, + "node-name": "#block112", + "backing_file_depth": 0, + "drv": "raw", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "/home/sven/Downloads/debian-8.5.0-amd64-netinst.iso", + "encryption_key_missing": false + }, + "tray_open": false, + "type": "unknown" + }, { + "io-status": "ok", + "device": "ide0-hd0", + "locked": false, + "removable": false, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 5368709120, + "filename": "test.qcow", + "cluster-size": 65536, + "format": "qcow2", + "actual-size": 1939349504, + "format-specific": { + "type": "qcow2", + "data": { + "compat": "1.1", + "lazy-refcounts": false, + "refcount-bits": 16, + "corrupt": false + } + }, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": false, + "node-name": "#block321", + "backing_file_depth": 0, + "drv": "qcow2", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "test.qcow", + "encryption_key_missing": false + }, + "type": "unknown" + }, { + "device": "floppy0", + "locked": false, + "removable": true, + "tray_open": true, + "type": "unknown" + }, { + "device": "sd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }] +} + +{ + "return": [{ + "io-status": "ok", + "device": "ide0-hd0", + "locked": false, + "removable": false, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 5368709120, + "filename": "test.qcow", + "cluster-size": 65536, + "format": "qcow2", + "actual-size": 1939349504, + "format-specific": { + "type": "qcow2", + "data": { + "compat": "1.1", + "lazy-refcounts": false, + "refcount-bits": 16, + "corrupt": false + } + }, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": false, + "node-name": "#block121", + "backing_file_depth": 0, + "drv": "qcow2", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "test.qcow", + "encryption_key_missing": false + }, + "type": "unknown" + }, { + "io-status": "ok", + "device": "ide1-cd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }, { + "device": "floppy0", + "locked": false, + "removable": true, + "tray_open": true, + "type": "unknown" + }, { + "device": "sd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }] +} + + + +{ + "return": [{ + "name": "query-rocker-of-dpa-groups" + }, { + "name": "query-rocker-of-dpa-flows" + }, { + "name": "query-rocker-ports" + }, { + "name": "query-rocker" + }, { + "name": "block-set-write-threshold" + }, { + "name": "x-input-send-event" + }, { + "name": "trace-event-set-state" + }, { + "name": "trace-event-get-state" + }, { + "name": "rtc-reset-reinjection" + }, { + "name": "query-acpi-ospm-status" + }, { + "name": "query-memory-devices" + }, { + "name": "query-memdev" + }, { + "name": "blockdev-change-medium" + }, { + "name": "query-named-block-nodes" + }, { + "name": "x-blockdev-insert-medium" + }, { + "name": "x-blockdev-remove-medium" + }, { + "name": "blockdev-close-tray" + }, { + "name": "blockdev-open-tray" + }, { + "name": "x-blockdev-del" + }, { + "name": "blockdev-add" + }, { + "name": "query-rx-filter" + }, { + "name": "chardev-remove" + }, { + "name": "chardev-add" + }, { + "name": "query-tpm-types" + }, { + "name": "query-tpm-models" + }, { + "name": "query-tpm" + }, { + "name": "query-target" + }, { + "name": "query-cpu-definitions" + }, { + "name": "query-machines" + }, { + "name": "device-list-properties" + }, { + "name": "qom-list-types" + }, { + "name": "change-vnc-password" + }, { + "name": "nbd-server-stop" + }, { + "name": "nbd-server-add" + }, { + "name": "nbd-server-start" + }, { + "name": "qom-get" + }, { + "name": "qom-set" + }, { + "name": "qom-list" + }, { + "name": "query-block-jobs" + }, { + "name": "query-balloon" + }, { + "name": "query-migrate-parameters" + }, { + "name": "migrate-set-parameters" + }, { + "name": "query-migrate-capabilities" + }, { + "name": "migrate-set-capabilities" + }, { + "name": "query-migrate" + }, { + "name": "query-command-line-options" + }, { + "name": "query-uuid" + }, { + "name": "query-name" + }, { + "name": "query-spice" + }, { + "name": "query-vnc-servers" + }, { + "name": "query-vnc" + }, { + "name": "query-mice" + }, { + "name": "query-status" + }, { + "name": "query-kvm" + }, { + "name": "query-pci" + }, { + "name": "query-iothreads" + }, { + "name": "query-cpus" + }, { + "name": "query-blockstats" + }, { + "name": "query-block" + }, { + "name": "query-chardev-backends" + }, { + "name": "query-chardev" + }, { + "name": "query-qmp-schema" + }, { + "name": "query-events" + }, { + "name": "query-commands" + }, { + "name": "query-version" + }, { + "name": "human-monitor-command" + }, { + "name": "qmp_capabilities" + }, { + "name": "add_client" + }, { + "name": "expire_password" + }, { + "name": "set_password" + }, { + "name": "block_set_io_throttle" + }, { + "name": "block_passwd" + }, { + "name": "query-fdsets" + }, { + "name": "remove-fd" + }, { + "name": "add-fd" + }, { + "name": "closefd" + }, { + "name": "getfd" + }, { + "name": "set_link" + }, { + "name": "balloon" + }, { + "name": "change-backing-file" + }, { + "name": "drive-mirror" + }, { + "name": "blockdev-snapshot-delete-internal-sync" + }, { + "name": "blockdev-snapshot-internal-sync" + }, { + "name": "blockdev-snapshot" + }, { + "name": "blockdev-snapshot-sync" + }, { + "name": "block-dirty-bitmap-clear" + }, { + "name": "block-dirty-bitmap-remove" + }, { + "name": "block-dirty-bitmap-add" + }, { + "name": "transaction" + }, { + "name": "block-job-complete" + }, { + "name": "block-job-resume" + }, { + "name": "block-job-pause" + }, { + "name": "block-job-cancel" + }, { + "name": "block-job-set-speed" + }, { + "name": "blockdev-backup" + }, { + "name": "drive-backup" + }, { + "name": "block-commit" + }, { + "name": "block-stream" + }, { + "name": "block_resize" + }, { + "name": "object-del" + }, { + "name": "object-add" + }, { + "name": "netdev_del" + }, { + "name": "netdev_add" + }, { + "name": "query-dump-guest-memory-capability" + }, { + "name": "dump-guest-memory" + }, { + "name": "client_migrate_info" + }, { + "name": "migrate_set_downtime" + }, { + "name": "migrate_set_speed" + }, { + "name": "query-migrate-cache-size" + }, { + "name": "migrate-start-postcopy" + }, { + "name": "migrate-set-cache-size" + }, { + "name": "migrate-incoming" + }, { + "name": "migrate_cancel" + }, { + "name": "migrate" + }, { + "name": "xen-set-global-dirty-log" + }, { + "name": "xen-save-devices-state" + }, { + "name": "ringbuf-read" + }, { + "name": "ringbuf-write" + }, { + "name": "inject-nmi" + }, { + "name": "pmemsave" + }, { + "name": "memsave" + }, { + "name": "cpu-add" + }, { + "name": "cpu" + }, { + "name": "send-key" + }, { + "name": "device_del" + }, { + "name": "device_add" + }, { + "name": "system_powerdown" + }, { + "name": "system_reset" + }, { + "name": "system_wakeup" + }, { + "name": "cont" + }, { + "name": "stop" + }, { + "name": "screendump" + }, { + "name": "change" + }, { + "name": "eject" + }, { + "name": "quit" + }] +} diff --git a/notes/qmp-cdrom-mounted.json b/notes/qmp-cdrom-mounted.json new file mode 100644 index 0000000..3a31589 --- /dev/null +++ b/notes/qmp-cdrom-mounted.json @@ -0,0 +1,96 @@ +{ + "return": [{ + "io-status": "ok", + "device": "ide1-cd0", + "locked": false, + "removable": true, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 258998272, + "filename": "/home/sven/Downloads/debian-8.5.0-amd64-netinst.iso", + "format": "raw", + "actual-size": 259006464, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": true, + "node-name": "#block112", + "backing_file_depth": 0, + "drv": "raw", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "/home/sven/Downloads/debian-8.5.0-amd64-netinst.iso", + "encryption_key_missing": false + }, + "tray_open": false, + "type": "unknown" + }, { + "io-status": "ok", + "device": "ide0-hd0", + "locked": false, + "removable": false, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 5368709120, + "filename": "test.qcow", + "cluster-size": 65536, + "format": "qcow2", + "actual-size": 1939349504, + "format-specific": { + "type": "qcow2", + "data": { + "compat": "1.1", + "lazy-refcounts": false, + "refcount-bits": 16, + "corrupt": false + } + }, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": false, + "node-name": "#block321", + "backing_file_depth": 0, + "drv": "qcow2", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "test.qcow", + "encryption_key_missing": false + }, + "type": "unknown" + }, { + "device": "floppy0", + "locked": false, + "removable": true, + "tray_open": true, + "type": "unknown" + }, { + "device": "sd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }] +} diff --git a/notes/qmp-cdrom-unmounted.js b/notes/qmp-cdrom-unmounted.js new file mode 100644 index 0000000..bb3f583 --- /dev/null +++ b/notes/qmp-cdrom-unmounted.js @@ -0,0 +1,67 @@ +{ + "return": [{ + "io-status": "ok", + "device": "ide0-hd0", + "locked": false, + "removable": false, + "inserted": { + "iops_rd": 0, + "detect_zeroes": "off", + "image": { + "virtual-size": 5368709120, + "filename": "test.qcow", + "cluster-size": 65536, + "format": "qcow2", + "actual-size": 1939349504, + "format-specific": { + "type": "qcow2", + "data": { + "compat": "1.1", + "lazy-refcounts": false, + "refcount-bits": 16, + "corrupt": false + } + }, + "dirty-flag": false + }, + "iops_wr": 0, + "ro": false, + "node-name": "#block121", + "backing_file_depth": 0, + "drv": "qcow2", + "iops": 0, + "bps_wr": 0, + "write_threshold": 0, + "encrypted": false, + "bps": 0, + "bps_rd": 0, + "cache": { + "no-flush": false, + "direct": false, + "writeback": true + }, + "file": "test.qcow", + "encryption_key_missing": false + }, + "type": "unknown" + }, { + "io-status": "ok", + "device": "ide1-cd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }, { + "device": "floppy0", + "locked": false, + "removable": true, + "tray_open": true, + "type": "unknown" + }, { + "device": "sd0", + "locked": false, + "removable": true, + "tray_open": false, + "type": "unknown" + }] +} diff --git a/notes/qmp-commands.json b/notes/qmp-commands.json new file mode 100644 index 0000000..9e8e864 --- /dev/null +++ b/notes/qmp-commands.json @@ -0,0 +1,273 @@ +{ + "return": [{ + "name": "query-rocker-of-dpa-groups" + }, { + "name": "query-rocker-of-dpa-flows" + }, { + "name": "query-rocker-ports" + }, { + "name": "query-rocker" + }, { + "name": "block-set-write-threshold" + }, { + "name": "x-input-send-event" + }, { + "name": "trace-event-set-state" + }, { + "name": "trace-event-get-state" + }, { + "name": "rtc-reset-reinjection" + }, { + "name": "query-acpi-ospm-status" + }, { + "name": "query-memory-devices" + }, { + "name": "query-memdev" + }, { + "name": "blockdev-change-medium" + }, { + "name": "query-named-block-nodes" + }, { + "name": "x-blockdev-insert-medium" + }, { + "name": "x-blockdev-remove-medium" + }, { + "name": "blockdev-close-tray" + }, { + "name": "blockdev-open-tray" + }, { + "name": "x-blockdev-del" + }, { + "name": "blockdev-add" + }, { + "name": "query-rx-filter" + }, { + "name": "chardev-remove" + }, { + "name": "chardev-add" + }, { + "name": "query-tpm-types" + }, { + "name": "query-tpm-models" + }, { + "name": "query-tpm" + }, { + "name": "query-target" + }, { + "name": "query-cpu-definitions" + }, { + "name": "query-machines" + }, { + "name": "device-list-properties" + }, { + "name": "qom-list-types" + }, { + "name": "change-vnc-password" + }, { + "name": "nbd-server-stop" + }, { + "name": "nbd-server-add" + }, { + "name": "nbd-server-start" + }, { + "name": "qom-get" + }, { + "name": "qom-set" + }, { + "name": "qom-list" + }, { + "name": "query-block-jobs" + }, { + "name": "query-balloon" + }, { + "name": "query-migrate-parameters" + }, { + "name": "migrate-set-parameters" + }, { + "name": "query-migrate-capabilities" + }, { + "name": "migrate-set-capabilities" + }, { + "name": "query-migrate" + }, { + "name": "query-command-line-options" + }, { + "name": "query-uuid" + }, { + "name": "query-name" + }, { + "name": "query-spice" + }, { + "name": "query-vnc-servers" + }, { + "name": "query-vnc" + }, { + "name": "query-mice" + }, { + "name": "query-status" + }, { + "name": "query-kvm" + }, { + "name": "query-pci" + }, { + "name": "query-iothreads" + }, { + "name": "query-cpus" + }, { + "name": "query-blockstats" + }, { + "name": "query-block" + }, { + "name": "query-chardev-backends" + }, { + "name": "query-chardev" + }, { + "name": "query-qmp-schema" + }, { + "name": "query-events" + }, { + "name": "query-commands" + }, { + "name": "query-version" + }, { + "name": "human-monitor-command" + }, { + "name": "qmp_capabilities" + }, { + "name": "add_client" + }, { + "name": "expire_password" + }, { + "name": "set_password" + }, { + "name": "block_set_io_throttle" + }, { + "name": "block_passwd" + }, { + "name": "query-fdsets" + }, { + "name": "remove-fd" + }, { + "name": "add-fd" + }, { + "name": "closefd" + }, { + "name": "getfd" + }, { + "name": "set_link" + }, { + "name": "balloon" + }, { + "name": "change-backing-file" + }, { + "name": "drive-mirror" + }, { + "name": "blockdev-snapshot-delete-internal-sync" + }, { + "name": "blockdev-snapshot-internal-sync" + }, { + "name": "blockdev-snapshot" + }, { + "name": "blockdev-snapshot-sync" + }, { + "name": "block-dirty-bitmap-clear" + }, { + "name": "block-dirty-bitmap-remove" + }, { + "name": "block-dirty-bitmap-add" + }, { + "name": "transaction" + }, { + "name": "block-job-complete" + }, { + "name": "block-job-resume" + }, { + "name": "block-job-pause" + }, { + "name": "block-job-cancel" + }, { + "name": "block-job-set-speed" + }, { + "name": "blockdev-backup" + }, { + "name": "drive-backup" + }, { + "name": "block-commit" + }, { + "name": "block-stream" + }, { + "name": "block_resize" + }, { + "name": "object-del" + }, { + "name": "object-add" + }, { + "name": "netdev_del" + }, { + "name": "netdev_add" + }, { + "name": "query-dump-guest-memory-capability" + }, { + "name": "dump-guest-memory" + }, { + "name": "client_migrate_info" + }, { + "name": "migrate_set_downtime" + }, { + "name": "migrate_set_speed" + }, { + "name": "query-migrate-cache-size" + }, { + "name": "migrate-start-postcopy" + }, { + "name": "migrate-set-cache-size" + }, { + "name": "migrate-incoming" + }, { + "name": "migrate_cancel" + }, { + "name": "migrate" + }, { + "name": "xen-set-global-dirty-log" + }, { + "name": "xen-save-devices-state" + }, { + "name": "ringbuf-read" + }, { + "name": "ringbuf-write" + }, { + "name": "inject-nmi" + }, { + "name": "pmemsave" + }, { + "name": "memsave" + }, { + "name": "cpu-add" + }, { + "name": "cpu" + }, { + "name": "send-key" + }, { + "name": "device_del" + }, { + "name": "device_add" + }, { + "name": "system_powerdown" + }, { + "name": "system_reset" + }, { + "name": "system_wakeup" + }, { + "name": "cont" + }, { + "name": "stop" + }, { + "name": "screendump" + }, { + "name": "change" + }, { + "name": "eject" + }, { + "name": "quit" + }] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c3adfb5 --- /dev/null +++ b/package.json @@ -0,0 +1,67 @@ +{ + "name": "cvm", + "version": "1.0.0", + "description": "A VPS management panel", + "main": "index.js", + "scripts": { + "watch": "gulp watch", + "gulp": "gulp", + "knex": "knex" + }, + "repository": { + "type": "git", + "url": "git@git.cryto.net:cvm" + }, + "author": "Sven Slootweg", + "license": "WTFPL", + "dependencies": { + "@joepie91/gulp-partial-patch-livereload-logger": "^1.0.1", + "JSONStream": "^1.1.4", + "assure-array": "^1.0.0", + "bhttp": "^1.2.4", + "bluebird": "^3.4.6", + "body-parser": "^1.15.2", + "checkit": "^0.7.0", + "create-error": "^0.3.1", + "create-event-emitter": "^1.0.0", + "debounce": "^1.0.0", + "default-value": "^1.0.0", + "end-of-stream": "^1.1.0", + "express": "^4.14.0", + "express-promise-router": "^1.1.0", + "express-ws": "^3.0.0", + "fs-extra": "^3.0.1", + "knex": "^0.13.0", + "pg": "^6.1.0", + "pug": "^2.0.0-beta6", + "rfr": "^1.2.3", + "scrypt-for-humans": "^2.0.5", + "split": "^1.0.0", + "through2": "^2.0.1", + "uuid": "^2.0.2" + }, + "devDependencies": { + "@joepie91/gulp-preset-es2015": "^1.0.1", + "@joepie91/gulp-preset-scss": "^1.0.1", + "babel-core": "^6.14.0", + "babel-loader": "^6.4.1", + "babel-preset-es2015": "^6.14.0", + "babel-preset-es2015-riot": "^1.1.0", + "chokidar": "^1.6.0", + "gulp": "^3.9.1", + "gulp-cached": "^1.1.0", + "gulp-livereload": "^3.8.1", + "gulp-named-log": "^1.0.1", + "gulp-nodemon": "^2.1.0", + "gulp-rename": "^1.2.2", + "jade": "^1.11.0", + "json-loader": "^0.5.4", + "listening": "^0.1.0", + "nodemon": "^1.10.2", + "riot": "^3.6.1", + "riotjs-loader": "^4.0.0", + "tiny-lr": "^0.2.1", + "webpack": "^1.15.0", + "webpack-stream": "^3.2.0" + } +} diff --git a/public/bundle.js b/public/bundle.js new file mode 100644 index 0000000..6ab43c8 --- /dev/null +++ b/public/bundle.js @@ -0,0 +1,50 @@ +/******/ (function(modules) { // webpackBootstrap +/******/ // The module cache +/******/ var installedModules = {}; + +/******/ // The require function +/******/ function __webpack_require__(moduleId) { + +/******/ // Check if module is in cache +/******/ if(installedModules[moduleId]) +/******/ return installedModules[moduleId].exports; + +/******/ // Create a new module (and put it into the cache) +/******/ var module = installedModules[moduleId] = { +/******/ exports: {}, +/******/ id: moduleId, +/******/ loaded: false +/******/ }; + +/******/ // Execute the module function +/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); + +/******/ // Flag the module as loaded +/******/ module.loaded = true; + +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } + + +/******/ // expose the modules object (__webpack_modules__) +/******/ __webpack_require__.m = modules; + +/******/ // expose the module cache +/******/ __webpack_require__.c = installedModules; + +/******/ // __webpack_public_path__ +/******/ __webpack_require__.p = ""; + +/******/ // Load entry module and return exports +/******/ return __webpack_require__(0); +/******/ }) +/************************************************************************/ +/******/ ([ +/* 0 */ +/***/ (function(module, exports) { + + 'use strict'; + +/***/ }) +/******/ ]); \ No newline at end of file diff --git a/public/css/style.css b/public/css/style.css new file mode 100644 index 0000000..0b0e72e --- /dev/null +++ b/public/css/style.css @@ -0,0 +1,36 @@ +body { + background-color: #e4e4e4; + margin: 0px; + font-family: sans-serif; } + +.content { + padding: 8px; } + +label { + margin-right: 12px; } + +.menu { + background-color: #000424; } + .menu h1, .menu .menu-item { + display: inline-block; + color: white; } + .menu h1 { + margin: 0px 16px; } + .menu .menu-item a { + color: white; + text-decoration: none; + padding: 15px 9px 5px 9px; } + .menu .menu-item.active a { + background-color: #e4e4e4; + color: black; } + .menu .menu-item:not(.active) a:hover { + background-color: #afafaf; + color: black; } + +table { + border-collapse: collapse; } + table th, table td { + padding: 6px 9px; + border: 1px solid black; } + table th { + text-align: left; } diff --git a/routes/disk-images.js b/routes/disk-images.js new file mode 100644 index 0000000..76b0f98 --- /dev/null +++ b/routes/disk-images.js @@ -0,0 +1,70 @@ +'use strict'; + +const Promise = require("bluebird"); +const checkit = require("checkit"); +const debounce = require("debounce"); + +const validatorAdd = require("../lib/form-validators/disk-images/add"); + +module.exports = function({db, imageStore}) { + let router = require("express-promise-router")(); + + router.get("/", (req, res) => { + return Promise.try(() => { + return db("images"); + }).then((results) => { + res.render("disk-images/list", { + images: results + }); + }); + }); + + router.get("/add", (req, res) => { + res.render("disk-images/add"); + }); + + router.post("/add", (req, res) => { + return Promise.try(() => { + return validatorAdd.run(req.body); + }).then((result) => { + /* FIXME: Only allow 'local' for administrators! */ + if (req.body.source === "http") { + return Promise.try(() => { + return imageStore.addFromUrl(req.body.url); + }).then((downloadTracker) => { + downloadTracker.on("progress", debounce((progress, description) => { + console.log("Download progress:", (Math.round(progress * 10000) / 100), "%"); + }, 200)); + + return downloadTracker.await(); + }); + } else if (req.body.source === "local") { + return imageStore.addFromPath(req.body.path); + } + }).then((imageId) => { + let source; + + if (req.body.source === "http") { + source = req.body.url; + } else if (req.body.source === "local") { + source = req.body.path; + } + + return db("images").insert({ + userId: 0, + fileId: imageId, + name: req.body.name, + description: req.body.description, + sourceType: req.body.source, + source: source, + imageType: "disk", + public: true, + isInstallMedium: true + }); + }).then(() => { + res.redirect("/disk-images"); + }); + }); + + return router; +} diff --git a/routes/index.js b/routes/index.js new file mode 100644 index 0000000..2e12aea --- /dev/null +++ b/routes/index.js @@ -0,0 +1,9 @@ +'use strict'; + +let router = require("express-promise-router")(); + +router.get("/", (req, res) => { + res.render("index"); +}); + +module.exports = router; diff --git a/routes/instances.js b/routes/instances.js new file mode 100644 index 0000000..fb9bea7 --- /dev/null +++ b/routes/instances.js @@ -0,0 +1,27 @@ +'use strict'; + +const Promise = require("bluebird"); + +module.exports = function({db}) { + let router = require("express-promise-router")(); + + router.get("/", (req, res) => { + return Promise.try(() => { + return db("instances"); + }).then((instances) => { + res.render("instances/list", { + instances: instances + }); + }); + }); + + router.get("/add", (req, res) => { + res.render("instances/add"); + }); + + router.post("/add", (req, res) => { + + }) + + return router; +} diff --git a/routes/tasks.js b/routes/tasks.js new file mode 100644 index 0000000..e02c1d9 --- /dev/null +++ b/routes/tasks.js @@ -0,0 +1,112 @@ +'use strict'; + +const rfr = require("rfr"); + +const errors = rfr("lib/errors"); +const tagWebsocket = rfr("lib/tag-websocket"); +const tagMessage = rfr("lib/tag-message"); +const websocketTracker = rfr("lib/websockets/tracker"); +const websocketDeduplicator = rfr("lib/deduplicator"); +const fakeTask = rfr("lib/tasks/fake-task"); + +module.exports = function({taskTracker}) { + let router = require("express-promise-router")(); + + /* We manually create a deduplicator, so that we can share it across all the user-level websocket trackers. This does mean that we have to manually remove disconnected clients from our shared deduplicator. */ + let deduplicator = websocketDeduplicator(); + + let allClients = websocketTracker({deduplicator: deduplicator}); + let userClients = websocketTracker({deduplicator: deduplicator, namespaced: true}); + let roleClients = websocketTracker({deduplicator: deduplicator, namespaced: true}); + + /* The adminClient tracker is a special case, and can use its own internal deduplicator if needed */ + let adminClients = websocketTracker(); + + function emitTask(task, message) { + task.userIds.forEach((userId) => { + userClients.emit(userId, message); + }); + + task.roleIds.forEach((roleId) => { + roleClients.emit(roleId, message); + }); + + adminClients.emit(message); + } + + taskTracker.on("newTask", (task) => { + emitTask(task, tagMessage({ + // FIXME: Task state? pending/started/paused/cancelled/etc. + taskId: task.id, + type: "newTask", + name: task.name, + progress: task.progress, + started: task.started, + lastUpdated: task.lastUpdated, + lastOperation: task.lastOperation + })); + }); + + taskTracker.on("progress", (task, progress, lastOperation) => { + emitTask(task, tagMessage({ + taskId: task.id, + type: "progress", + progress: progress, + lastOperation: lastOperation + })); + }); + + taskTracker.on("completed", (task) => { + emitTask(tagMessage({ + taskId: task.id, + type: "completed" + })); + }); + + // FIXME: Initial task list + + router.ws("/feed", (ws, req) => { + // FIXME: Auth user? + if (req.session.user == null) { + throw new errors.UnauthorizedError("You must be authenticated to obtain a task feed"); + } else { + tagWebsocket(ws); + // We ignore client messages for now + + allClients.add(client); + userClients.add(req.session.user.id, client); + + req.session.user.roles.forEach((role) => { + roleClients.add(role.id, client); + }); + + ws.on("close", () => { + allClients.remove(client); + userClients.remove(client); + roleClients.remove(client); + deduplicator.forgetClient(client); + }); + } + }); + + router.ws("/feed/all", (ws, req) => { + // FIXME: Require admin access + tagWebsocket(ws); + // We ignore client messages for now + + adminClients.add(client); + + ws.on("close", () => { + adminClients.remove(client); + }); + }); + + router.get("/fake-task", (req, res) => { + let task = fakeTask(5000); + addTask("Fake Task", task, [req.session.user.id], [], {}); + + res.send("Fake task added!"); + }) + + return router; +} diff --git a/scss/style.scss b/scss/style.scss new file mode 100644 index 0000000..352a841 --- /dev/null +++ b/scss/style.scss @@ -0,0 +1,65 @@ +body { + background-color: rgb(228, 228, 228); + margin: 0px; + font-family: sans-serif; +} + +.content { + padding: 8px; +} + +.form-section { + +} + +label { + margin-right: 12px; +} + +.menu { + background-color: rgb(0, 4, 36); + + h1, .menu-item { + display: inline-block; + color: white; + } + + h1 { + margin: 0px 16px; + } + + .menu-item { + a { + color: white; + text-decoration: none; + padding: 15px 9px 5px 9px; + } + + &.active { + a { + background-color: rgb(228, 228, 228); + color: black; + } + } + + &:not(.active) { + a:hover { + background-color: rgb(175, 175, 175); + color: black; + } + } + } +} + +table { + border-collapse: collapse; + + th, td { + padding: 6px 9px; + border: 1px solid black; + } + + th { + text-align: left; + } +} diff --git a/views/disk-images/add.pug b/views/disk-images/add.pug new file mode 100644 index 0000000..89620a3 --- /dev/null +++ b/views/disk-images/add.pug @@ -0,0 +1,40 @@ +extends ../layout + +block content + h2 From URL + form(method="post", action="/disk-images/add") + input(type="hidden", name="source", value="http") + + .form-section + label Name + input(type="text", name="name") + + .form-section + label Description + input(type="text", name="description") + + .form-section + label URL + input(type="text", name="url") + + .form-section + button(type="submit") Add + + h2 From disk + form(method="post", action="/disk-images/add") + input(type="hidden", name="source", value="local") + + .form-section + label Name + input(type="text", name="name") + + .form-section + label Description + input(type="text", name="description") + + .form-section + label Path to image + input(type="text", name="path") + + .form-section + button(type="submit") Add diff --git a/views/disk-images/list.pug b/views/disk-images/list.pug new file mode 100644 index 0000000..baa81da --- /dev/null +++ b/views/disk-images/list.pug @@ -0,0 +1,19 @@ +extends ../layout + +block content + h2 Disk Images + + a(href="/disk-images/add") Add + + table + tr + th ID + th Name + th Description + th Source + for image in images + tr + td= image.id + td: strong= image.name + td= image.description + td= image.source diff --git a/views/error.pug b/views/error.pug new file mode 100644 index 0000000..10129a3 --- /dev/null +++ b/views/error.pug @@ -0,0 +1,6 @@ +extends layout + +block content + h1 An error occurred. + h2= error.message + pre= error.stack diff --git a/views/index.pug b/views/index.pug new file mode 100644 index 0000000..5d0d4c0 --- /dev/null +++ b/views/index.pug @@ -0,0 +1,4 @@ +extends layout + +block content + | Hello World! diff --git a/views/instances/list.pug b/views/instances/list.pug new file mode 100644 index 0000000..fea5acb --- /dev/null +++ b/views/instances/list.pug @@ -0,0 +1,25 @@ +extends ../layout + +block content + table + tr + th ID + th User ID + th Identifier + th Type + th Memory + th Swap + th Disk + th Traffic + th Running? + for instance in instances + tr + th= instance.id + th= instance.userId + th= instance.customIdentifier + th= instance.virtualizationType + th= instance.memory + th= instance.swap + th= instance.diskSpace + th= instance.traffic + th= instance.running diff --git a/views/layout.pug b/views/layout.pug new file mode 100644 index 0000000..dac0f80 --- /dev/null +++ b/views/layout.pug @@ -0,0 +1,17 @@ +mixin menu-item(prefix) + .menu-item(class=isUnderPrefix(prefix, "active")) + block + +doctype html +head + title CVM + link(rel="stylesheet", href="/css/style.css") +body + .menu + h1 CVM + +menu-item("/disk-images"): a(href="/disk-images") Disk Images + +menu-item("/instances"): a(href="/instances") Instances + +menu-item("/users"): a(href="/users") Users + + .content + block content