WIP, metrics
parent
90ef557a30
commit
522192f025
@ -0,0 +1,3 @@
|
||||
# srap
|
||||
|
||||
An unopinionated tag-based scraping server. Documentation coming soon™.
|
||||
@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
"use strict";
|
||||
|
||||
// let realConsoleLog = console.log.bind(console);
|
||||
// console.log = function(...args) {
|
||||
// realConsoleLog(`[console.log called from ${(new Error).stack.split("\n")[2].replace(/^\s* at /, "")}]`);
|
||||
// realConsoleLog(...args);
|
||||
// };
|
||||
|
||||
// let realConsoleError = console.error.bind(console);
|
||||
// console.error = function(...args) {
|
||||
// realConsoleError(`[console.error called from ${(new Error).stack.split("\n")[2].replace(/^\s* at /, "")}]`);
|
||||
// realConsoleError(...args);
|
||||
// };
|
||||
|
||||
// FIXME: All of this is a work-in-progress, zero stability guarantees!
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const express = require("express");
|
||||
const expressPromiseRouter = require("express-promise-router");
|
||||
const assert = require("assert");
|
||||
const path = require("path");
|
||||
|
||||
const pipe = require("@promistream/pipe");
|
||||
const fromNodeStream = require("@promistream/from-node-stream");
|
||||
const map = require("@promistream/map");
|
||||
|
||||
const { testValue } = require("@validatem/core");
|
||||
const matchesFormat = require("@validatem/matches-format");
|
||||
const isString = require("@validatem/is-string");
|
||||
|
||||
const initialize = require("../src/initialize");
|
||||
const errors = require("../src/errors");
|
||||
|
||||
assert(process.argv.length >= 4);
|
||||
let configurationPath = process.argv[2];
|
||||
let listenHost = process.argv[3];
|
||||
|
||||
let absoluteConfigurationPath = path.join(process.cwd(), configurationPath);
|
||||
let configuration = require(absoluteConfigurationPath);
|
||||
|
||||
return Promise.try(() => {
|
||||
// FIXME: Deduplicate this with kernel! Also other common wiring across binaries...
|
||||
return initialize({
|
||||
knexfile: {
|
||||
client: "pg",
|
||||
connection: configuration.database,
|
||||
pool: { min: 0, max: 32 },
|
||||
migrations: { tableName: "srap_knex_migrations" }
|
||||
}
|
||||
});
|
||||
}).then((state) => {
|
||||
let { db, knex } = state;
|
||||
|
||||
const queries = require("../src/queries")(state);
|
||||
|
||||
let app = express();
|
||||
let router = expressPromiseRouter();
|
||||
|
||||
// router.get("/items/:id", (req, res) => {
|
||||
// return Promise.try(() => {
|
||||
// return db.Item.query(knex).findById(req.params.id);
|
||||
// }).then((item) => {
|
||||
// if (item != null) {
|
||||
// res.json(item);
|
||||
// } else {
|
||||
// throw new errors.NotFound(`No such item exists`);
|
||||
// }
|
||||
// });
|
||||
// });
|
||||
|
||||
// router.delete("/items/:id", (req, res) => {
|
||||
// return Promise.try(() => {
|
||||
// return db.Item.query(knex)
|
||||
// .findById(req.params.id)
|
||||
// .delete();
|
||||
// }).then((affectedRows) => {
|
||||
// if (affectedRows > 0) {
|
||||
// res.status(204).end();
|
||||
// } else {
|
||||
// throw new errors.NotFound(`No such item exists`);
|
||||
// }
|
||||
// });
|
||||
// });
|
||||
|
||||
// // MARKER: Stub out routes, replace error implementation, add response generation for HTTP errors
|
||||
|
||||
// router.get("/items/:id/metadata", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
// router.get("/items/:id/operations", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
// router.get("/items/:id/operations/:operation/expire", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
// router.post("/items/add", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
// router.put("/items/add/:id", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
// router.get("/items/:id", (req, res) => {
|
||||
|
||||
// });
|
||||
|
||||
router.get("/updates", (req, res) => {
|
||||
// FIXME: Proper Express integration for Validatem
|
||||
let isValid = testValue(req.query, {
|
||||
since: matchesFormat(/^[0-9]+$/),
|
||||
prefix: isString
|
||||
});
|
||||
|
||||
if (isValid) {
|
||||
let timestamp = (req.query.since != null)
|
||||
? new Date(parseInt(req.query.since))
|
||||
: undefined;
|
||||
|
||||
return pipe([
|
||||
queries.getUpdates(knex, { prefix: req.query.prefix, timestamp: timestamp }),
|
||||
map((item) => JSON.stringify(item) + "\n"),
|
||||
fromNodeStream(res)
|
||||
]).read();
|
||||
} else {
|
||||
res.status(422).send("Invalid request");
|
||||
}
|
||||
});
|
||||
|
||||
app.use(router);
|
||||
app.listen({ host: listenHost, port: 3000 }, () => {
|
||||
console.log("Server listening on port 3000");
|
||||
});
|
||||
});
|
||||
@ -1,109 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
// let realConsoleLog = console.log.bind(console);
|
||||
// console.log = function(...args) {
|
||||
// realConsoleLog(`[console.log called from ${(new Error).stack.split("\n")[2].replace(/^\s* at /, "")}]`);
|
||||
// realConsoleLog(...args);
|
||||
// };
|
||||
|
||||
// let realConsoleError = console.error.bind(console);
|
||||
// console.error = function(...args) {
|
||||
// realConsoleError(`[console.error called from ${(new Error).stack.split("\n")[2].replace(/^\s* at /, "")}]`);
|
||||
// realConsoleError(...args);
|
||||
// };
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const express = require("express");
|
||||
const expressPromiseRouter = require("express-promise-router");
|
||||
|
||||
const pipe = require("@promistream/pipe");
|
||||
const fromNodeStream = require("@promistream/from-node-stream");
|
||||
const map = require("@promistream/map");
|
||||
|
||||
const initialize = require("./initialize");
|
||||
const errors = require("./errors");
|
||||
|
||||
return Promise.try(() => {
|
||||
return initialize({
|
||||
knexfile: require("../knexfile")
|
||||
});
|
||||
}).then((state) => {
|
||||
let { db, knex } = state;
|
||||
|
||||
const queries = require("./queries")(state);
|
||||
|
||||
let app = express();
|
||||
let router = expressPromiseRouter();
|
||||
|
||||
router.get("/items/:id", (req, res) => {
|
||||
return Promise.try(() => {
|
||||
return db.Item.query(knex).findById(req.params.id);
|
||||
}).then((item) => {
|
||||
if (item != null) {
|
||||
res.json(item);
|
||||
} else {
|
||||
throw new errors.NotFound(`No such item exists`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
router.delete("/items/:id", (req, res) => {
|
||||
return Promise.try(() => {
|
||||
return db.Item.query(knex)
|
||||
.findById(req.params.id)
|
||||
.delete();
|
||||
}).then((affectedRows) => {
|
||||
if (affectedRows > 0) {
|
||||
res.status(204).end();
|
||||
} else {
|
||||
throw new errors.NotFound(`No such item exists`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// MARKER: Stub out routes, replace error implementation, add response generation for HTTP errors
|
||||
|
||||
router.get("/items/:id/metadata", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.get("/items/:id/operations", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.get("/items/:id/operations/:operation/expire", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.post("/items/add", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.put("/items/add/:id", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.get("/items/:id", (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
router.get("/updates", (req, res) => {
|
||||
let timestamp = (req.query.since != null)
|
||||
? new Date(parseInt(req.query.since))
|
||||
: undefined;
|
||||
|
||||
console.log({ prefix: req.query.prefix, timestamp: timestamp });
|
||||
|
||||
// FIXME: Validation!
|
||||
return pipe([
|
||||
queries.getUpdates(knex, { prefix: req.query.prefix, timestamp: timestamp }),
|
||||
map((item) => JSON.stringify(item) + "\n"),
|
||||
fromNodeStream(res)
|
||||
]).read();
|
||||
});
|
||||
|
||||
app.use(router);
|
||||
app.listen(3000, () => {
|
||||
console.log("Server listening on port 3000");
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue