Browse Source

WIP: Various web-y things.

feature/node-rewrite
Sven Slootweg 1 year ago
parent
commit
e10935c79a
25 changed files with 558 additions and 263 deletions
  1. +0
    -52
      app.js
  2. +1
    -47
      bin/server.js
  3. +5
    -2
      lib/disk-store.js
  4. +0
    -16
      lib/errors.js
  5. +0
    -21
      lib/form-validators/disk-images/add.js
  6. +1
    -1
      lib/tasks/tracker.js
  7. +36
    -0
      public/css/style.css
  8. +0
    -112
      routes/tasks.js
  9. +66
    -0
      src/app.js
  10. +17
    -0
      src/auth/middleware.js
  11. +3
    -0
      src/client/index.jsx
  12. +24
    -0
      src/db/disk-images.js
  13. +0
    -0
      src/image-store.js
  14. +15
    -0
      src/pools/storage/file.js
  15. +11
    -11
      src/routes/disk-images.js
  16. +0
    -0
      src/routes/index.js
  17. +0
    -0
      src/routes/instances.js
  18. +97
    -0
      src/routes/storage-devices.js
  19. +114
    -0
      src/routes/tasks.js
  20. +61
    -0
      src/scss/style.scss
  21. +3
    -1
      src/tag-message.js
  22. +0
    -0
      src/tasks/progress-indicator.js
  23. +31
    -0
      src/validators/disk-images/add.js
  24. +70
    -0
      views/hardware/storage-devices/list.pug
  25. +3
    -0
      views/layout.pug

+ 0
- 52
app.js View File

@@ -1,52 +0,0 @@
'use strict';

const express = require("express");
const expressWs = require("express-ws");
const knex = require("knex");
const path = require("path");
const bodyParser = require("body-parser");

let db = knex(require("./knexfile"));
let imageStore = require("./lib/image-store")(path.join(__dirname, "./images"));
let taskTracker = require("./lib/tasks/tracker")();

let state = {db, imageStore, taskTracker};

let app = express();
expressWs(app);

app.set("view engine", "pug");
app.set("views", path.join(__dirname, "views"));

app.use((req, res, next) => {
res.locals.isUnderPrefix = function isUnderPrefix(path, resultingClass) {
// FIXME: Proper path segment parsing...
if (req.originalUrl.indexOf(path) === 0) {
return resultingClass;
} else {
return "";
}
}

next();
});

app.use(express.static(path.join(__dirname, "public")));

app.use(bodyParser.urlencoded({
extended: true
}));

app.use(require("./routes/index"));
app.use("/disk-images", require("./routes/disk-images")(state));
app.use("/instances", require("./routes/instances")(state));

app.use((err, req, res, next) => {
res.render("error", {
error: err
});
});

app.listen(3000).on("listening", () => {
console.log("Listening...");
});

+ 1
- 47
bin/server.js View File

@@ -24,50 +24,4 @@ budoExpress({
}]
]
},
});


// const app = require("../src/app.js")();

// if (process.env.NODE_ENV === "development") {
// const budo = require("budo");

// function projectPath(targetPath) {
// return path.resolve(__dirname, "..", targetPath);
// }

// budo(projectPath("src/client/index.jsx"), {
// watchGlob: projectPath("public/css/*.css"),
// live: "**/*.css",
// stream: process.stdout,
// port: 8000,
// dir: projectPath("public"),
// serve: "js/bundle.js",
// debug: true,
// browserify: {
// extensions: [".jsx"],
// plugin: [
// "browserify-hmr"
// ],
// transform: [
// ["babelify", {
// presets: ["@babel/preset-env", "@babel/preset-react"],
// plugins: ["react-hot-loader/babel"]
// }]
// ]
// },
// middleware: function (req, res, next) {
// app.handle(req, res, (err) => {
// if (err != null && err instanceof Error) {
// res.send("<pre>" + err.stack + "</pre>");
// } else {
// next(err);
// }
// });
// }
// });
// } else {
// app.listen(3000).on("listening", () => {
// console.log("Listening...");
// });
// }
});

+ 5
- 2
lib/disk-store.js View File

@@ -2,6 +2,7 @@

const Promise = require("bluebird");
const uuid = require("uuid");
const defaultValue = require("default-value");
const childProcess = Promise.promisifyAll(require("child-process"), {multiArgs: true});

module.exports = function(storePath) {
@@ -13,14 +14,16 @@ module.exports = function(storePath) {
getPath: function getDisk(id) {
return getPath(id);
},
create: function createDisk(size, {type} = {type: "qcow2"}) {
create: function createDisk(size, options = {}) {
return Promise.try(() => {
let imageFormat = defaultValue(options.format, "qcow2");

let diskId = uuid.v4();

return Promise.try(() => {
childProcess.execFileAsync("qemu-img", [
"create",
"-f", type,
"-f", imageFormat,
getPath(diskId),
size
]);


+ 0
- 16
lib/errors.js View File

@@ -1,16 +0,0 @@
'use strict';

const createError = require("create-error");

let HttpError = createError("HttpError", {
exposeToUser: true
});

module.exports = {
UnauthorizedError: createError(HttpError, "UnauthorizedError", {
statusCode: 401
}),
ForbiddenError: createError(HttpError, "ForbiddenError", {
statusCode: 403
})
}

+ 0
- 21
lib/form-validators/disk-images/add.js View File

@@ -1,21 +0,0 @@
'use strict';

const checkit = require("checkit");
const oneOf = require("../../validators/one-of");

module.exports = checkit({
name: "string",
description: "string",
source: ["required", "string", oneOf([
"local",
"http"
])]
}).maybe({
url: ["required", "string"]
}, (input) => {
return (input.source === "http");
}).maybe({
path: ["required", "string"]
}, (input) => {
return (input.source === "local");
});

+ 1
- 1
lib/tasks/tracker.js View File

@@ -9,7 +9,7 @@ const findAndSplice = require("../find-and-splice");
module.exports = function createTaskTracker() {
let tasks = [];

return createEventEmitter{
return createEventEmitter({
addTask: function addTask(name, indicator, userIds, roleIds, options) {
let mergedOptions = Object.assign({
removeWhenDone: true


+ 36
- 0
public/css/style.css View File

@@ -34,3 +34,39 @@ table {
border: 1px solid black; }
table th {
text-align: left; }
table td.hidden {
border: none; }

table.drives td.smart.healthy {
background-color: #00a500; }

table.drives td.smart.deteriorating {
background-color: #ff9100; }

table.drives td.smart.failing {
background-color: #e60000; }

table.drives .hasPartitions td:not(.smart), table.drives .partition:not(.last) td:not(.smart) {
border-bottom-color: transparent; }

table.drives .partition {
font-style: italic;
font-size: .8em; }
table.drives .partition td {
padding: 4px 9px; }
table.drives .partition .notMounted {
color: gray; }

table.drives tr.smartStatus {
font-size: .85em; }
table.drives tr.smartStatus td {
padding: 4px 9px; }

table.drives th.healthy {
color: #006100; }

table.drives th.atRisk {
color: #7c4600; }

table.drives th.failing {
color: #c20000; }

+ 0
- 112
routes/tasks.js View File

@@ -1,112 +0,0 @@
'use strict';

const rfr = require("rfr");

const errors = rfr("lib/errors");
const tagWebsocket = rfr("lib/tag-websocket");
const tagMessage = rfr("lib/tag-message");
const websocketTracker = rfr("lib/websockets/tracker");
const websocketDeduplicator = rfr("lib/deduplicator");
const fakeTask = rfr("lib/tasks/fake-task");

module.exports = function({taskTracker}) {
let router = require("express-promise-router")();

/* We manually create a deduplicator, so that we can share it across all the user-level websocket trackers. This does mean that we have to manually remove disconnected clients from our shared deduplicator. */
let deduplicator = websocketDeduplicator();

let allClients = websocketTracker({deduplicator: deduplicator});
let userClients = websocketTracker({deduplicator: deduplicator, namespaced: true});
let roleClients = websocketTracker({deduplicator: deduplicator, namespaced: true});

/* The adminClient tracker is a special case, and can use its own internal deduplicator if needed */
let adminClients = websocketTracker();

function emitTask(task, message) {
task.userIds.forEach((userId) => {
userClients.emit(userId, message);
});

task.roleIds.forEach((roleId) => {
roleClients.emit(roleId, message);
});

adminClients.emit(message);
}

taskTracker.on("newTask", (task) => {
emitTask(task, tagMessage({
// FIXME: Task state? pending/started/paused/cancelled/etc.
taskId: task.id,
type: "newTask",
name: task.name,
progress: task.progress,
started: task.started,
lastUpdated: task.lastUpdated,
lastOperation: task.lastOperation
}));
});

taskTracker.on("progress", (task, progress, lastOperation) => {
emitTask(task, tagMessage({
taskId: task.id,
type: "progress",
progress: progress,
lastOperation: lastOperation
}));
});

taskTracker.on("completed", (task) => {
emitTask(tagMessage({
taskId: task.id,
type: "completed"
}));
});

// FIXME: Initial task list

router.ws("/feed", (ws, req) => {
// FIXME: Auth user?
if (req.session.user == null) {
throw new errors.UnauthorizedError("You must be authenticated to obtain a task feed");
} else {
tagWebsocket(ws);
// We ignore client messages for now

allClients.add(client);
userClients.add(req.session.user.id, client);

req.session.user.roles.forEach((role) => {
roleClients.add(role.id, client);
});

ws.on("close", () => {
allClients.remove(client);
userClients.remove(client);
roleClients.remove(client);
deduplicator.forgetClient(client);
});
}
});

router.ws("/feed/all", (ws, req) => {
// FIXME: Require admin access
tagWebsocket(ws);
// We ignore client messages for now

adminClients.add(client);

ws.on("close", () => {
adminClients.remove(client);
});
});

router.get("/fake-task", (req, res) => {
let task = fakeTask(5000);
addTask("Fake Task", task, [req.session.user.id], [], {});

res.send("Fake task added!");
})

return router;
}

+ 66
- 0
src/app.js View File

@@ -0,0 +1,66 @@
'use strict';

const express = require("express");
// const expressWs = require("express-ws");
const knex = require("knex");
const path = require("path");
const bodyParser = require("body-parser");

function projectPath(targetPath) {
return path.join(__dirname, "..", targetPath);
}

module.exports = function () {
let db = knex(require("../knexfile"));
let imageStore = require("./image-store")(projectPath("./images"));
let taskTracker = require("../lib/tasks/tracker")();

let state = {db, imageStore, taskTracker};

let app = express();
// expressWs(app);

app.set("view engine", "pug");
app.set("views", projectPath("views"));

app.use((req, res, next) => {
res.locals.isUnderPrefix = function isUnderPrefix(path, resultingClass) {
// FIXME: Proper path segment parsing...
if (req.originalUrl.indexOf(path) === 0) {
return resultingClass;
} else {
return "";
}
}

next();
});

app.use(express.static(projectPath("public")));

app.use(bodyParser.urlencoded({
extended: true
}));

app.use(require("./routes/index"));
app.use("/disk-images", require("./routes/disk-images")(state));
app.use("/instances", require("./routes/instances")(state));
app.use("/hardware/storage-devices", require("./routes/storage-devices")(state));

app.use((err, req, res, next) => {
if (err.showChain != null) {
console.log(err.showChain());
console.log("#####################");
console.log(err.getAllContext());
} else {
console.log(err.stack);
}

res.render("error", {
error: err
});
});

return app;
};

+ 17
- 0
src/auth/middleware.js View File

@@ -0,0 +1,17 @@
"use strict";

const errors = require("../errors");

module.exports = {
isAuthenticated: function (req, res, next) {
if (req.session.user != null) {
next();
} else {
throw new errors.UnauthorizedError("You must be authenticated to view this page");
}
},
isAdministrator: function (req, res, next) {
/* FIXME */
next();
}
};

+ 3
- 0
src/client/index.jsx View File

@@ -0,0 +1,3 @@
"use strict";

console.log("Hello world!");

+ 24
- 0
src/db/disk-images.js View File

@@ -0,0 +1,24 @@
"use strict";

const Promise = require("bluebird");

module.exports = function ({db}) {
return {
getAll: function () {
return db("images");
},
addInstallISO: function ({userId, fileId, name, description, sourceType, source, isPublic}) {
return db("images").insert({
userId: userId,
fileId: fileId,
name: name,
description: description,
sourceType: sourceType,
source: source,
public: isPublic,
imageType: "disk",
isInstallMedium: true
});
}
};
};

lib/image-store.js → src/image-store.js View File


+ 15
- 0
src/pools/storage/file.js View File

@@ -0,0 +1,15 @@
"use strict";

module.exports = function createFileStoragePool({ storePath }) {
return {
createDisk: function ({ size, format }) {
},
resizeDisk: function ({ id, newSize }) {
},
removeDisk: function ({ id }) {
}
};
};

routes/disk-images.js → src/routes/disk-images.js View File

@@ -1,17 +1,18 @@
'use strict';

const Promise = require("bluebird");
const checkit = require("checkit");
const debounce = require("debounce");
const functionRateLimit = require("function-rate-limit");

const validatorAdd = require("../lib/form-validators/disk-images/add");
const validatorAdd = require("../validators/disk-images/add");

module.exports = function({db, imageStore}) {
const dbDiskImages = require("../db/disk-images")({db});

let router = require("express-promise-router")();

router.get("/", (req, res) => {
return Promise.try(() => {
return db("images");
return dbDiskImages.getAll();
}).then((results) => {
res.render("disk-images/list", {
images: results
@@ -25,16 +26,17 @@ module.exports = function({db, imageStore}) {

router.post("/add", (req, res) => {
return Promise.try(() => {
return validatorAdd.run(req.body);
return validatorAdd.validate(req.body);
}).then((result) => {
/* FIXME: Only allow 'local' for administrators! */
if (req.body.source === "http") {
return Promise.try(() => {
return imageStore.addFromUrl(req.body.url);
}).then((downloadTracker) => {
downloadTracker.on("progress", debounce((progress, description) => {
downloadTracker.on("progress", functionRateLimit(1, 200, (progress, description) => {
/* FIXME: This is getting debounced to a point of pointlessness as it will only fire when the download is completed. Needs to be replaced by SSE events anyway, and rate-limited instead of debounced. */
console.log("Download progress:", (Math.round(progress * 10000) / 100), "%");
}, 200));
}));

return downloadTracker.await();
});
@@ -50,16 +52,14 @@ module.exports = function({db, imageStore}) {
source = req.body.path;
}

return db("images").insert({
return dbDiskImages.addInstallISO({
userId: 0,
fileId: imageId,
name: req.body.name,
description: req.body.description,
sourceType: req.body.source,
source: source,
imageType: "disk",
public: true,
isInstallMedium: true
isPublic: true
});
}).then(() => {
res.redirect("/disk-images");

routes/index.js → src/routes/index.js View File


routes/instances.js → src/routes/instances.js View File


+ 97
- 0
src/routes/storage-devices.js View File

@@ -0,0 +1,97 @@
'use strict';

const Promise = require("bluebird");

const lsblk = require("../wrappers/lsblk");
const smartctl = require("../wrappers/smartctl");
const lvm = require("../wrappers/lvm");
const {B} = require("../units/bytes/iec");

/* FIXME: Move this to GraphQL API */
function getSmartStatus(smartData) {
let failed = smartData.filter((item) => {
return (item.failingNow === true || item.failedBefore === true);
});

let deteriorating = smartData.filter((item) => {
return (item.type === "preFail" && item.worstValueSeen < 100);
});

if (failed.length > 0) {
return "failed";
} else if (deteriorating.length > 0) {
return "deteriorating";
} else {
return "healthy";
}
}

function getStorageDevices() {
return Promise.try(() => {
return lsblk();
}).filter((device) => {
return (device.type === "disk");
}).map((device) => {
return Object.assign({}, device, {
path: `/dev/${device.name}`
});
}).map((device) => {
/* FIXME: Check whether we need to iterate through child disks as well, when dealing with eg. RAID arrays */
return Promise.try(() => {
return Promise.all([
smartctl.info({ devicePath: device.path }),
smartctl.attributes({ devicePath: device.path })
]);
}).then(([info, attributes]) => {
return Object.assign({}, device, {
information: info,
smartData: attributes,
smartStatus: getSmartStatus(attributes)
});
});
}).then((blockDevices) => {
console.log(blockDevices);
return blockDevices;
});
}

function sumDriveSizes(drives) {
return drives.reduce((total, device) => {
return total + device.size.toB().amount;
}, 0);
}

function roundUnit(unit) {
return Object.assign(unit, {
amount: Math.round(unit.amount * 100) / 100
});
}

module.exports = function({db}) {
let router = require("express-promise-router")();

router.get("/", (req, res) => {
return Promise.try(() => {
return getStorageDevices();
}).then((devices) => {
/* FIXME: Auto-formatting of total sizes and units */
let fixedDrives = devices.filter((drive) => drive.removable === false);
let removableDrives = devices.filter((drive) => drive.removable === true);

let healthyFixedDrives = fixedDrives.filter((drive) => drive.smartStatus === "healthy");
let deterioratingFixedDrives = fixedDrives.filter((drive) => drive.smartStatus === "deteriorating");
let failingFixedDrives = fixedDrives.filter((drive) => drive.smartStatus === "failing");

res.render("hardware/storage-devices/list", {
devices: devices,
totalFixedStorage: roundUnit(B(sumDriveSizes(fixedDrives)).toTiB()),
totalHealthyFixedStorage: roundUnit(B(sumDriveSizes(healthyFixedDrives)).toTiB()),
totalDeterioratingFixedStorage: roundUnit(B(sumDriveSizes(deterioratingFixedDrives)).toTiB()),
totalFailingFixedStorage: roundUnit(B(sumDriveSizes(failingFixedDrives)).toTiB()),
totalRemovableStorage: roundUnit(B(sumDriveSizes(removableDrives)).toGiB())
});
});
});

return router;
}

+ 114
- 0
src/routes/tasks.js View File

@@ -0,0 +1,114 @@
'use strict';

const sseChannel = require("@joepie91/sse-channel");

const errors = require("../errors");
// const tagWebsocket = require("lib/tag-websocket");
const tagMessage = require("../tag-message");
// const websocketTracker = require("lib/websockets/tracker");
// const websocketDeduplicator = require("lib/deduplicator");
const fakeTask = require("lib/tasks/fake-task");

const authMiddleware = require("../auth/middleware");

module.exports = function({taskTracker}) {
let router = require("express-promise-router")();

// /* We manually create a deduplicator, so that we can share it across all the user-level websocket trackers. This does mean that we have to manually remove disconnected clients from our shared deduplicator. */
// let deduplicator = websocketDeduplicator();

// let allClients = websocketTracker({deduplicator: deduplicator});
// let userClients = websocketTracker({deduplicator: deduplicator, namespaced: true});
// let roleClients = websocketTracker({deduplicator: deduplicator, namespaced: true});

// /* The adminClient tracker is a special case, and can use its own internal deduplicator if needed */
// let adminClients = websocketTracker();

// function emitTask(task, message) {
// task.userIds.forEach((userId) => {
// userClients.emit(userId, message);
// });

// task.roleIds.forEach((roleId) => {
// roleClients.emit(roleId, message);
// });

// adminClients.emit(message);
// }

taskTracker.on("newTask", (task) => {
emitTask(task, tagMessage({
// FIXME: Task state? pending/started/paused/cancelled/etc.
taskId: task.id,
type: "newTask",
name: task.name,
progress: task.progress,
started: task.started,
lastUpdated: task.lastUpdated,
lastOperation: task.lastOperation
}));
});

taskTracker.on("progress", (task, progress, lastOperation) => {
emitTask(task, tagMessage({
taskId: task.id,
type: "progress",
progress: progress,
lastOperation: lastOperation
}));
});

taskTracker.on("completed", (task) => {
emitTask(tagMessage({
taskId: task.id,
type: "completed"
}));
});

// FIXME: Initial task list

router.get("/feed", authMiddleware.isAuthenticated, (req, res) => {
// FIXME: Auth user?
if (req.session.user == null) {
throw new errors.UnauthorizedError("You must be authenticated to obtain a task feed");
} else {
// tagWebsocket(ws);
// // We ignore client messages for now

// allClients.add(client);
// userClients.add(req.session.user.id, client);

// req.session.user.roles.forEach((role) => {
// roleClients.add(role.id, client);
// });

// ws.on("close", () => {
// allClients.remove(client);
// userClients.remove(client);
// roleClients.remove(client);
// deduplicator.forgetClient(client);
// });
}
});

router.get("/feed/all", authMiddleware.isAdministrator, (req, res) => {
// FIXME: Require admin access
tagWebsocket(ws);
// We ignore client messages for now

adminClients.add(client);

ws.on("close", () => {
adminClients.remove(client);
});
});

router.get("/fake-task", (req, res) => {
let task = fakeTask(5000);
addTask("Fake Task", task, [req.session.user.id], [], {});

res.send("Fake task added!");
});

return router;
}

scss/style.scss → src/scss/style.scss View File

@@ -62,4 +62,65 @@ table {
th {
text-align: left;
}

td.hidden {
border: none;
}
}

table.drives {
td.smart {
&.healthy {
background-color: rgb(0, 165, 0);
}

&.deteriorating {
background-color: rgb(255, 145, 0);
}

&.failing {
background-color: rgb(230, 0, 0);
}
}

.hasPartitions, .partition:not(.last) {
td:not(.smart) {
border-bottom-color: transparent;
}
}

.partition {
font-style: italic;
font-size: .8em;

td {
padding: 4px 9px;
}

.notMounted {
color: gray;
}
}

tr.smartStatus {
font-size: .85em;

td {
padding: 4px 9px;
}
}

th {
&.healthy {
color: rgb(0, 97, 0);
}

&.atRisk {
color: rgb(124, 70, 0);
}

&.failing {
color: rgb(194, 0, 0);
}
}
}

lib/tag-message.js → src/tag-message.js View File

@@ -2,9 +2,11 @@

const uuid = require("uuid");

let i = 0;

module.exports = function tagMessage(message) {
if (message.messageId == null) {
message.messageId = uuid.v4();
message.messageId = i++;
}

return message;

lib/tasks/progress-indicator.js → src/tasks/progress-indicator.js View File


+ 31
- 0
src/validators/disk-images/add.js View File

@@ -0,0 +1,31 @@
'use strict';

const joi = require("joi");

// const checkit = require("checkit");
// const oneOf = require("../../validators/one-of");

module.exports = joi.object({
name: joi.string().required(),
description: joi.string(),
source: joi.string().required(),
url: joi.when("source", { is: "http", then: joi.string().required() }),
path: joi.when("source", { is: "local", then: joi.string().required() })
});

// module.exports = checkit({
// name: "string",
// description: "string",
// source: ["required", "string", oneOf([
// "local",
// "http"
// ])]
// }).maybe({
// url: ["required", "string"]
// }, (input) => {
// return (input.source === "http");
// }).maybe({
// path: ["required", "string"]
// }, (input) => {
// return (input.source === "local");
// });

+ 70
- 0
views/hardware/storage-devices/list.pug View File

@@ -0,0 +1,70 @@
extends ../../layout

block content

h2 Fixed drives

//- FIXME: Partitions with mountpoints
table.drives
tr
th SMART
th Device
th Total size
th RPM
th Serial number
th Model
th Family
th Firmware version
for device in devices.filter((device) => device.removable === false)
tr(class=(device.children.length > 0 ? "hasPartitions" : null))
td(class=`smart ${device.smartStatus}`, rowspan=(1 + device.children.length))
td= device.name
td= device.size
td #{device.information.rpm} RPM
td= device.information.serialNumber
td= device.information.model
td= device.information.modelFamily
td= device.information.firmwareVersion

for partition, i in device.children
tr.partition(class=(i === device.children.length - 1) ? "last" : null)
td= partition.name
td= partition.size
td(colspan=5)
if partition.mountpoint != null
= partition.mountpoint
else
span.notMounted (not mounted)

//- tr.partition
//- td(colspan=8)= JSON.stringify(partition)
tr
th(colspan=2) Total
td= totalFixedStorage
td(colspan=5).hidden
tr.smartStatus
th(colspan=2).healthy Healthy
td= totalHealthyFixedStorage
td(colspan=5).hidden
tr.smartStatus
th(colspan=2).atRisk At-risk
td= totalDeterioratingFixedStorage
td(colspan=5).hidden
tr.smartStatus
th(colspan=2).failing Failing
td= totalFailingFixedStorage
td(colspan=5).hidden

h2 Removable drives

table
tr
th Path
th Total size
th Mounted at
for device in devices.filter((device) => device.type === "loopDevice")
tr
td= device.path
td= device.size
td= device.mountpoint

+ 3
- 0
views/layout.pug View File

@@ -15,3 +15,6 @@ body

.content
block content

script(src="/js/bundle.js")
script(src="/budo/livereload.js")

Loading…
Cancel
Save