Initial commit
commit
8a9864ea81
@ -0,0 +1,3 @@
|
||||
errors
|
||||
node_modules
|
||||
config.json
|
||||
@ -0,0 +1,9 @@
|
||||
{
|
||||
"scraperSettings": {
|
||||
"pastebinCom": {
|
||||
"listInterval": 60,
|
||||
"listLimit": 100,
|
||||
"pasteInterval": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
const Promise = require("bluebird");
|
||||
const path = require("path");
|
||||
const gulp = require("gulp");
|
||||
const webpackStream = require("webpack-stream");
|
||||
const webpack = require("webpack");
|
||||
const gulpNamedLog = require("gulp-named-log");
|
||||
const gulpRename = require("gulp-rename");
|
||||
const gulpNodemon = require("gulp-nodemon");
|
||||
const gulpCached = require("gulp-cached");
|
||||
const presetSCSS = require("@joepie91/gulp-preset-scss");
|
||||
|
||||
const awaitServer = require("await-server");
|
||||
|
||||
const gulpLivereload = require("gulp-livereload");
|
||||
const patchLivereloadLogger = require("@joepie91/gulp-partial-patch-livereload-logger");
|
||||
|
||||
patchLivereloadLogger(gulpLivereload);
|
||||
|
||||
let config = {
|
||||
scss: {
|
||||
source: "./src/scss/**/*.scss",
|
||||
destination: "./public/css/"
|
||||
},
|
||||
cssModules: []
|
||||
}
|
||||
|
||||
let serverLogger = gulpNamedLog("server");
|
||||
|
||||
gulp.task("nodemon", ["scss", "livereload"], () => {
|
||||
gulpNodemon({
|
||||
script: "server.js",
|
||||
ignore: [
|
||||
"gulpfile.js",
|
||||
"node_modules",
|
||||
"public"
|
||||
],
|
||||
ext: "js pug"
|
||||
}).on("start", () => {
|
||||
Promise.try(() => {
|
||||
serverLogger.info("Starting...");
|
||||
return awaitServer(3000);
|
||||
}).then(() => {
|
||||
serverLogger.info("Started!");
|
||||
gulpLivereload.changed("*");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
gulp.task("scss", () => {
|
||||
return gulp.src(config.scss.source)
|
||||
.pipe(presetSCSS({
|
||||
livereload: gulpLivereload
|
||||
}))
|
||||
.pipe(gulp.dest(config.scss.destination));
|
||||
});
|
||||
|
||||
gulp.task("livereload", () => {
|
||||
gulpLivereload.listen({
|
||||
quiet: true
|
||||
});
|
||||
});
|
||||
|
||||
gulp.task("webpack-watch", ["livereload"], () => {
|
||||
return gulp.src("./src/frontend/index.js")
|
||||
.pipe(webpackStream({
|
||||
watch: true,
|
||||
module: {
|
||||
preLoaders: [{
|
||||
test: /\.tag$/,
|
||||
loader: "riotjs-loader",
|
||||
exclude: /node_modules/,
|
||||
query: {
|
||||
type: "es6",
|
||||
template: "pug",
|
||||
parserOptions: {
|
||||
js: {
|
||||
presets: ["es2015-riot"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}],
|
||||
loaders: [{
|
||||
test: /\.js/,
|
||||
loader: "babel-loader",
|
||||
query: {
|
||||
presets: ["es2015"]
|
||||
}
|
||||
}, {
|
||||
test: /\.json$/,
|
||||
loader: "json-loader"
|
||||
}]
|
||||
},
|
||||
resolve: {
|
||||
extensions: [
|
||||
"",
|
||||
".tag",
|
||||
".web.js", ".js",
|
||||
".web.json", ".json"
|
||||
]
|
||||
},
|
||||
plugins: [ new webpack.ProvidePlugin({riot: "riot"}) ],
|
||||
debug: true
|
||||
}))
|
||||
.pipe(gulpRename("bundle.js"))
|
||||
.pipe(gulpNamedLog("webpack").stream())
|
||||
.pipe(gulpLivereload())
|
||||
.pipe(gulp.dest("./public/js/"));
|
||||
});
|
||||
|
||||
gulp.task("watch-css", () => {
|
||||
gulp.watch(config.scss.source, ["scss"]);
|
||||
});
|
||||
|
||||
gulp.task("watch", ["webpack-watch", "nodemon", "watch-css"]);
|
||||
|
||||
gulp.task("default", ["watch"]);
|
||||
@ -0,0 +1,53 @@
|
||||
{
|
||||
"name": "pastebin-stream",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git@git.cryto.net:joepie91/pastebin-stream.git"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "Sven Slootweg",
|
||||
"license": "WTFPL",
|
||||
"dependencies": {
|
||||
"bhttp": "^1.2.4",
|
||||
"bluebird": "^3.5.0",
|
||||
"create-error": "^0.3.1",
|
||||
"create-event-emitter": "^1.0.0",
|
||||
"debug": "^2.6.3",
|
||||
"default-value": "^1.0.0",
|
||||
"document-ready-promise": "^3.0.1",
|
||||
"express": "^4.15.2",
|
||||
"express-ws": "^3.0.0",
|
||||
"mkdirp": "^0.5.1",
|
||||
"moment": "^2.18.0",
|
||||
"promise-task-queue": "^1.2.0",
|
||||
"pug": "^2.0.0-beta11",
|
||||
"report-errors": "^1.0.0",
|
||||
"ws": "^2.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@joepie91/gulp-partial-patch-livereload-logger": "^1.0.1",
|
||||
"@joepie91/gulp-preset-scss": "^1.0.3",
|
||||
"babel-core": "^6.22.1",
|
||||
"babel-loader": "^6.2.10",
|
||||
"babel-preset-es2015": "^6.22.0",
|
||||
"babel-preset-es2015-riot": "^1.1.0",
|
||||
"document-ready-promise": "^3.0.1",
|
||||
"gulp": "^3.9.1",
|
||||
"gulp-cached": "^1.1.1",
|
||||
"gulp-livereload": "^3.8.1",
|
||||
"gulp-named-log": "^1.0.1",
|
||||
"gulp-nodemon": "^2.2.1",
|
||||
"gulp-rename": "^1.2.2",
|
||||
"riot": "^3.3.2",
|
||||
"riotjs-loader": "^4.0.0",
|
||||
"webpack": "^1.14.0",
|
||||
"webpack-stream": "^3.2.0",
|
||||
"wsurl": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
body {
|
||||
font-family: sans-serif;
|
||||
line-height: 1.4; }
|
||||
|
||||
.wrapper {
|
||||
max-width: 900px;
|
||||
margin: 0px auto; }
|
||||
|
||||
pre {
|
||||
padding: 16px;
|
||||
border: 1px solid #bababa;
|
||||
background-color: #eeeeee;
|
||||
border-radius: 5px; }
|
||||
|
||||
code {
|
||||
padding: 3px 7px 3px 7px; }
|
||||
|
||||
code, pre {
|
||||
font-size: 16px;
|
||||
background-color: #eeeeee;
|
||||
border-radius: 5px; }
|
||||
|
||||
.donation-method {
|
||||
padding: 16px;
|
||||
border: 1px solid #bababa;
|
||||
background-color: #eeeeee;
|
||||
border-radius: 5px;
|
||||
margin-bottom: 8px; }
|
||||
.donation-method h3 {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 7px;
|
||||
border-bottom: none; }
|
||||
.donation-method p {
|
||||
margin: 5px 0px; }
|
||||
.donation-method .amount {
|
||||
font-size: 16px; }
|
||||
.donation-method .amount label {
|
||||
margin-right: 7px; }
|
||||
.donation-method .amount input {
|
||||
margin: 0px 2px;
|
||||
font-size: 16px;
|
||||
padding: 4px 5px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid silver;
|
||||
width: 80px; }
|
||||
.donation-method.monthly .amount input {
|
||||
width: 55px; }
|
||||
.donation-method .amount, .donation-method .submit {
|
||||
display: inline; }
|
||||
.donation-method .submit {
|
||||
position: relative;
|
||||
top: 8px;
|
||||
left: 12px; }
|
||||
|
||||
h2 {
|
||||
border-bottom: 1px solid silver; }
|
||||
|
||||
h3 {
|
||||
border-bottom: 1px solid #cdcdcd; }
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,81 @@
|
||||
'use strict';
|
||||
|
||||
const errorReporter = require("./src/error-reporting");
|
||||
|
||||
const path = require("path");
|
||||
const express = require("express");
|
||||
const expressWs = require("express-ws");
|
||||
|
||||
const config = require("./config.json");
|
||||
const clientStore = require("./src/client-store")();
|
||||
|
||||
const scraper = require("./src/scraper")({
|
||||
scrapePastebinCom: true,
|
||||
scraperSettings: config.scraperSettings
|
||||
});
|
||||
|
||||
scraper.on("newPaste", (paste) => {
|
||||
clientStore.broadcast({
|
||||
type: "newPaste",
|
||||
data: paste
|
||||
});
|
||||
});
|
||||
scraper.on("warning", (error) => {
|
||||
console.error(error);
|
||||
});
|
||||
|
||||
scraper.on("error", (error) => {
|
||||
errorReporter.report(error, {
|
||||
source: "scraper"
|
||||
});
|
||||
});
|
||||
|
||||
clientStore.on("error", (error) => {
|
||||
errorReporter.report(error, {
|
||||
source: "clientStore"
|
||||
});
|
||||
});
|
||||
|
||||
let app = express();
|
||||
|
||||
expressWs(app);
|
||||
|
||||
app.set("views", path.join(__dirname, "views"));
|
||||
app.set("view engine", "pug");
|
||||
|
||||
app.use(express.static(path.join(__dirname, "public")));
|
||||
|
||||
app.get("/", (req, res) => {
|
||||
res.render("index");
|
||||
});
|
||||
|
||||
app.ws("/stream", (ws, req) => {
|
||||
let client = clientStore.add(ws);
|
||||
|
||||
client.on("message", (message) => {
|
||||
if (message.type === "backlog") {
|
||||
if (message.all === true) {
|
||||
client.send({
|
||||
type: "backlog",
|
||||
results: scraper.pasteStore.all(message.since)
|
||||
});
|
||||
} else if (typeof message.since === "number") {
|
||||
client.send({
|
||||
type: "backlog",
|
||||
results: scraper.pasteStore.since(message.since)
|
||||
});
|
||||
} else if (typeof message.last === "number") {
|
||||
client.send({
|
||||
type: "backlog",
|
||||
results: scraper.pasteStore.last(message.last)
|
||||
});
|
||||
} else {
|
||||
client.kill();
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
app.listen(3000, () => {
|
||||
console.log("Listening on port 3000...");
|
||||
});
|
||||
@ -0,0 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function clampZero(value) {
|
||||
if (value < 0) {
|
||||
return 0;
|
||||
} else {
|
||||
return value;
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,102 @@
|
||||
'use strict';
|
||||
|
||||
const createEventEmitter = require("create-event-emitter");
|
||||
const defaultValue = require("default-value");
|
||||
|
||||
function removeFromArray(array, item) {
|
||||
let index = array.indexOf(item);
|
||||
|
||||
if (index !== -1) {
|
||||
array.splice(index, 1);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = function createClientStore(options = {}) {
|
||||
let clients = [];
|
||||
let subscribedClients = [];
|
||||
|
||||
let store = createEventEmitter({
|
||||
add: function addClient(client) {
|
||||
let clientObject = createEventEmitter({
|
||||
socket: client,
|
||||
subscribed: false,
|
||||
disconnected: false,
|
||||
kill: function killClient() {
|
||||
store.kill(client);
|
||||
},
|
||||
send: function sendMessage(message) {
|
||||
let data = JSON.stringify(message);
|
||||
this.socket.send(data, (error) => {
|
||||
if (error != null) {
|
||||
store.emit("error", error);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
client.on("message", (data) => {
|
||||
let message;
|
||||
|
||||
try {
|
||||
message = JSON.parse(data);
|
||||
} catch (err) {
|
||||
/* User sent invalid data; disconnect them. */
|
||||
return this.killClient(client);
|
||||
}
|
||||
|
||||
if (message == null || typeof message !== "object") {
|
||||
/* User sent invalid data; disconnect them. */
|
||||
return this.killClient(client);
|
||||
}
|
||||
|
||||
if (message.type === "subscribe") {
|
||||
if (clientObject.subscribed === false) {
|
||||
subscribedClients.push(clientObject);
|
||||
clientObject.subscribed = true;
|
||||
clientObject.emit("subscribed");
|
||||
}
|
||||
} else {
|
||||
clientObject.emit("message", message);
|
||||
}
|
||||
});
|
||||
|
||||
client.on("close", (code, reason) => {
|
||||
client.disconnected = true;
|
||||
this.remove(clientObject);
|
||||
});
|
||||
|
||||
clients.push(clientObject);
|
||||
|
||||
return clientObject;
|
||||
},
|
||||
kill: function killClient(client) {
|
||||
client.close();
|
||||
},
|
||||
remove: function removeClient(client) {
|
||||
removeFromArray(clients, client);
|
||||
removeFromArray(subscribedClients, client);
|
||||
},
|
||||
broadcast: function broadcastMessage(message, clients) {
|
||||
defaultValue(clients, subscribedClients).forEach((client) => {
|
||||
client.send(message);
|
||||
});
|
||||
},
|
||||
broadcastAll: function broadcastMessageToAll(message) {
|
||||
this.broadcast(message, clients);
|
||||
}
|
||||
});
|
||||
|
||||
let pingInterval = defaultValue(options.pingInterval, 5000);
|
||||
|
||||
function pingClients() {
|
||||
store.broadcastAll({
|
||||
type: "ping"
|
||||
});
|
||||
|
||||
setTimeout(pingClients, pingInterval);
|
||||
}
|
||||
|
||||
pingClients();
|
||||
|
||||
return store;
|
||||
};
|
||||
@ -0,0 +1,20 @@
|
||||
'use strict';
|
||||
|
||||
const path = require("path");
|
||||
const reportErrors = require("report-errors");
|
||||
const mkdirp = require("mkdirp");
|
||||
|
||||
let errorPath = path.join(__dirname, "../errors");
|
||||
|
||||
mkdirp.sync(errorPath);
|
||||
|
||||
module.exports = reportErrors(errorPath, {
|
||||
doNotCrash: true,
|
||||
handler: (error, context) => {
|
||||
console.error("Unhandled error", error.stack);
|
||||
|
||||
if (error.rateLimited !== true) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -0,0 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const createError = require("create-error");
|
||||
|
||||
module.exports = {
|
||||
HttpError: createError("HttpError")
|
||||
};
|
||||
@ -0,0 +1,91 @@
|
||||
app
|
||||
div.pasteBox
|
||||
div.paste(each="{ paste in pastes }")
|
||||
span.title(if="{ paste.title != null }") {paste.title}
|
||||
span.title(if="{ paste.title == null }") (no title)
|
||||
|
||||
span.date {paste.formattedDate}
|
||||
|
||||
span.url {paste.url}
|
||||
|
||||
script.
|
||||
const moment = require("moment");
|
||||
const wsEndpoint = require("../ws-endpoint");
|
||||
const pruneArray = require("../../prune-array");
|
||||
|
||||
function formatDate(date) {
|
||||
return moment.unix(date).format("HH:mm:ss");
|
||||
}
|
||||
|
||||
function addFormattedDate(paste) {
|
||||
return Object.assign({
|
||||
formattedDate: formatDate(paste.date)
|
||||
}, paste);
|
||||
}
|
||||
|
||||
this.pastes = [];
|
||||
|
||||
this.on("mount", () => {
|
||||
|
||||
let endpoint = wsEndpoint("/stream");
|
||||
let socket = new WebSocket(endpoint);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
let message = JSON.parse(event.data);
|
||||
|
||||
if (message.type === "newPaste") {
|
||||
let newPaste = message.data;
|
||||
this.pastes.push(addFormattedDate(newPaste));
|
||||
} else if (message.type === "backlog") {
|
||||
let unknownPastes = message.results.filter((paste) => {
|
||||
return !this.pastes.some((existingPaste) => (existingPaste.id === paste.id))
|
||||
}).map((paste) => {
|
||||
return addFormattedDate(paste);
|
||||
});
|
||||
|
||||
this.pastes = unknownPastes.concat(this.pastes);
|
||||
}
|
||||
|
||||
pruneArray(this.pastes, 10);
|
||||
this.update();
|
||||
}
|
||||
|
||||
socket.onopen = (event) => {
|
||||
socket.send(JSON.stringify({
|
||||
type: "backlog",
|
||||
last: 10
|
||||
}));
|
||||
|
||||
socket.send(JSON.stringify({
|
||||
type: "subscribe"
|
||||
}));
|
||||
}
|
||||
|
||||
this.update();
|
||||
});
|
||||
|
||||
style(type="scss").
|
||||
.pasteBox {
|
||||
height: 310px;
|
||||
border: 1px solid rgb(173, 173, 173);
|
||||
overflow: auto;
|
||||
|
||||
.paste {
|
||||
padding: 4px;
|
||||
background-color: rgb(209, 209, 209);
|
||||
border-bottom: 1px solid rgb(173, 173, 173);
|
||||
|
||||
.url {
|
||||
color: gray;
|
||||
}
|
||||
|
||||
.title {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.title:after, .date:after {
|
||||
content: "-";
|
||||
margin: 0px 4px;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const documentReadyPromise = require("document-ready-promise");
|
||||
const app = require("./components/app");
|
||||
const riot = require("riot");
|
||||
|
||||
Promise.try(() => {
|
||||
return documentReadyPromise();
|
||||
}).then((result) => {
|
||||
riot.mount("*");
|
||||
});
|
||||
@ -0,0 +1,8 @@
|
||||
'use strict';
|
||||
|
||||
const wsurl = require("wsurl");
|
||||
const url = require("url");
|
||||
|
||||
module.exports = function wsEndpoint(path) {
|
||||
return wsurl(url.resolve(document.location.href, path));
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
'use strict';
|
||||
|
||||
const defaultValue = require("default-value");
|
||||
const createEventEmitter = require("create-event-emitter");
|
||||
|
||||
const pruneArray = require("./prune-array");
|
||||
const clampZero = require("./clamp-zero");
|
||||
|
||||
module.exports = function createPasteStore(options = {}) {
|
||||
let pastes = [];
|
||||
let counter = 0;
|
||||
let pasteLimit = defaultValue(options.pasteLimit, 250);
|
||||
|
||||
return createEventEmitter({
|
||||
add: function addPaste(pasteData) {
|
||||
let paste = Object.assign({
|
||||
counter: counter++
|
||||
}, pasteData);
|
||||
|
||||
pastes.push(paste);
|
||||
this.emit("added", paste);
|
||||
|
||||
pruneArray(pastes, pasteLimit);
|
||||
},
|
||||
since: function getPastesSince(counter) {
|
||||
return pastes.filter((paste) => {
|
||||
return (paste.counter > counter);
|
||||
});
|
||||
},
|
||||
all: function getAllPastes() {
|
||||
return pastes;
|
||||
},
|
||||
last: function getLastPastes(count) {
|
||||
return pastes.slice(clampZero(pastes.length - count));
|
||||
}
|
||||
});
|
||||
};
|
||||
@ -0,0 +1,49 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const createEventEmitter = require("create-event-emitter");
|
||||
|
||||
const clampZero = require("./clamp-zero");
|
||||
|
||||
module.exports = function promiseSetInterval(callback, interval, options = {}) {
|
||||
let lastInvocation = Date.now();
|
||||
let timeout;
|
||||
|
||||
let emitter = createEventEmitter({
|
||||
cancel: function cancelInterval() {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
});
|
||||
|
||||
function scheduleNext() {
|
||||
let delta = Date.now() - lastInvocation;
|
||||
let timeout = clampZero(interval - delta);
|
||||
|
||||
scheduleInvocation(timeout);
|
||||
}
|
||||
|
||||
function scheduleInvocation(nextTimeout) {
|
||||
timeout = setTimeout(() => {
|
||||
Promise.try(() => {
|
||||
lastInvocation = Date.now();
|
||||
return callback();
|
||||
}).then(() => {
|
||||
scheduleNext();
|
||||
}).catch((err) => {
|
||||
emitter.emit("error", err);
|
||||
|
||||
if (!options.abortOnError) {
|
||||
scheduleNext();
|
||||
}
|
||||
});
|
||||
}, nextTimeout);
|
||||
}
|
||||
|
||||
if (options.startImmediately) {
|
||||
scheduleInvocation(0);
|
||||
} else {
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
return emitter;
|
||||
};
|
||||
@ -0,0 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
/* CAUTION: This mutates the array! */
|
||||
|
||||
module.exports = function pruneArray(array, maxItems) {
|
||||
if (array.length > maxItems) {
|
||||
array.splice(0, array.length - maxItems);
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,57 @@
|
||||
'use strict';
|
||||
|
||||
const createEventEmitter = require("create-event-emitter");
|
||||
const defaultValue = require("default-value");
|
||||
|
||||
function ifNotEmpty(value) {
|
||||
if (value !== "") {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = function createScraper(options = {}) {
|
||||
let scraperSettings = defaultValue(options.scraperSettings, {});
|
||||
|
||||
const pasteStore = require("./paste-store")();
|
||||
const pastebinComScraper = require("./scrapers/pastebin-com")(scraperSettings.pastebinCom);
|
||||
const errors = require("./errors");
|
||||
|
||||
let scraper = createEventEmitter({
|
||||
pasteStore: pasteStore
|
||||
});
|
||||
|
||||
pastebinComScraper.on("paste", (paste) => {
|
||||
let postedDate = parseInt(paste.date);
|
||||
let expiryDate = parseInt(paste.expire);
|
||||
|
||||
pasteStore.add({
|
||||
site: "pastebinCom",
|
||||
url: paste.full_url,
|
||||
date: postedDate,
|
||||
id: paste.key,
|
||||
expire: (expiryDate !== 0) ? (expiryDate - postedDate) : undefined,
|
||||
username: ifNotEmpty(paste.user),
|
||||
language: paste.syntax,
|
||||
title: ifNotEmpty(paste.title),
|
||||
contents: paste.raw
|
||||
});
|
||||
});
|
||||
|
||||
pastebinComScraper.on("error", (error) => {
|
||||
if (error instanceof errors.HttpError && error.rateLimited !== true) {
|
||||
scraper.emit("warning", error);
|
||||
} else {
|
||||
scraper.emit("error", error);
|
||||
}
|
||||
});
|
||||
|
||||
if (options.scrapePastebinCom) {
|
||||
pastebinComScraper.start();
|
||||
}
|
||||
|
||||
pasteStore.on("added", (paste) => {
|
||||
scraper.emit("newPaste", paste);
|
||||
});
|
||||
|
||||
return scraper;
|
||||
};
|
||||
@ -0,0 +1,93 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require("bluebird");
|
||||
const bhttp = require("bhttp");
|
||||
const promiseTaskQueue = require("promise-task-queue");
|
||||
const createEventEmitter = require("create-event-emitter");
|
||||
const defaultValue = require("default-value");
|
||||
const debug = require("debug")("pastebinStream:scrapers:pastebinCom");
|
||||
|
||||
const promiseSetInterval = require("../promise-set-interval");
|
||||
const errors = require("../errors");
|
||||
|
||||
function tryParseBody(body) {
|
||||
try {
|
||||
return JSON.parse(body);
|
||||
} catch (err) {
|
||||
throw new errors.HttpError(`Got rate-limited? Error message: ${body}`, {type: "rateLimited"});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = function createPastebinComScraper(options = {}) {
|
||||
let queue = promiseTaskQueue();
|
||||
let knownPastes = [];
|
||||
let previousKnownPastes = [];
|
||||
|
||||
queue.define("fetchPaste", (task) => {
|
||||
return Promise.try(() => {
|
||||
return bhttp.get(`http://pastebin.com/api_scrape_item.php?i=${task.pasteKey}`);
|
||||
}).then((response) => {
|
||||
if (response.statusCode !== 200) {
|
||||
// FIXME: Retry!
|
||||
throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while retrieving a raw paste: ${response.statusCode}`, {type: "statusCode"});
|
||||
} else if (response.body.toString().indexOf("Please slow down, you are hitting our servers unnecessarily hard!") === 0) {
|
||||
throw new errors.HttpError("Got rate-limited!", {type: "rateLimited"});
|
||||
} else {
|
||||
return response.body.toString();
|
||||
}
|
||||
})
|
||||
}, {
|
||||
interval: options.pasteInterval
|
||||
});
|
||||
|
||||
let loop;
|
||||
|
||||
return createEventEmitter({
|
||||
stop: function stopScraper() {
|
||||
if (loop != null) {
|
||||
this.emit("stopped");
|
||||
loop.cancel();
|
||||
}
|
||||
},
|
||||
start: function startScraper() {
|
||||
loop = promiseSetInterval(() => {
|
||||
return Promise.try(() => {
|
||||
return bhttp.get(`http://pastebin.com/api_scraping.php?limit=${defaultValue(options.listLimit, 100)}`, {
|
||||
noDecode: true /* Because Pastebin.com errors aren't JSON... */
|
||||
});
|
||||
}).then((response) => {
|
||||
if (response.statusCode !== 200) {
|
||||
throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while listing the most recent pastes: ${response.statusCode}`, {type: "statusCode"});
|
||||
} else {
|
||||
return tryParseBody(response.body).reverse();
|
||||
}
|
||||
}).tap((pastes) => {
|
||||
previousKnownPastes = knownPastes;
|
||||
knownPastes = pastes.map(paste => paste.key);
|
||||
}).filter((paste) => {
|
||||
return (!previousKnownPastes.includes(paste.key));
|
||||
}).tap((pastes) => {
|
||||
debug(`Found ${pastes.length} new pastes`);
|
||||
}).each((paste) => {
|
||||
/* We *intentionally* do not return the Promise chain below; we don't want to block the interval with queue items. */
|
||||
Promise.try(() => {
|
||||
return queue.push("fetchPaste", {
|
||||
pasteKey: paste.key
|
||||
});
|
||||
}).then((rawPaste) => {
|
||||
this.emit("paste", Object.assign({
|
||||
raw: rawPaste
|
||||
}, paste));
|
||||
}).catch((err) => {
|
||||
this.emit("error", err);
|
||||
});
|
||||
}).catch((err) => {
|
||||
/* This is where eg. rate-limiting errors will end up. */
|
||||
this.emit("error", err);
|
||||
});
|
||||
}, defaultValue(options.listInterval, 60) * 1000, {
|
||||
startImmediately: true
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
@ -0,0 +1,83 @@
|
||||
body {
|
||||
font-family: sans-serif;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.wrapper {
|
||||
max-width: 900px;
|
||||
margin: 0px auto;
|
||||
}
|
||||
|
||||
pre {
|
||||
padding: 16px;
|
||||
border: 1px solid rgb(186, 186, 186);
|
||||
background-color: rgb(238, 238, 238);
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
code {
|
||||
padding: 3px 7px 3px 7px;
|
||||
}
|
||||
|
||||
code, pre {
|
||||
font-size: 16px;
|
||||
background-color: #eeeeee;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
.donation-method {
|
||||
padding: 16px;
|
||||
border: 1px solid rgb(186, 186, 186);
|
||||
background-color: rgb(238, 238, 238);
|
||||
border-radius: 5px;
|
||||
margin-bottom: 8px;
|
||||
|
||||
h3 {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 7px;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 5px 0px;
|
||||
}
|
||||
|
||||
.amount {
|
||||
font-size: 16px;
|
||||
|
||||
label {
|
||||
margin-right: 7px;
|
||||
}
|
||||
|
||||
input {
|
||||
margin: 0px 2px;
|
||||
font-size: 16px;
|
||||
padding: 4px 5px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid silver;
|
||||
width: 80px;
|
||||
}
|
||||
}
|
||||
|
||||
&.monthly .amount input {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.amount, .submit {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
.submit {
|
||||
position: relative;
|
||||
top: 8px;
|
||||
left: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
h2 {
|
||||
border-bottom: 1px solid silver;
|
||||
}
|
||||
|
||||
h3 {
|
||||
border-bottom: 1px solid rgb(205, 205, 205);
|
||||
}
|
||||
@ -0,0 +1,225 @@
|
||||
doctype html
|
||||
html
|
||||
head
|
||||
title Pastebin Stream
|
||||
script(src="/js/bundle.js")
|
||||
link(rel="stylesheet", href="/css/style.css")
|
||||
body
|
||||
.wrapper
|
||||
h1 Pastebin Stream
|
||||
|
||||
p This is a simple streaming (WebSocket) API, providing a live feed of (public) pastes as they are submitted to <a href="http://pastebin.com/">Pastebin.com</a>. You can find the source code <a href="https://git.cryto.net/joepie91/pastebin-stream">here</a>.
|
||||
|
||||
p Since Pastebin does not offer a streaming API itself, this API works by regularly polling the Pastebin API for new posts. This means that posts may be delayed by about a minute, and that they'll be broadcast in 'batches'.
|
||||
|
||||
h2 Preview
|
||||
|
||||
p The latest pastes:
|
||||
|
||||
app
|
||||
|
||||
h2 Usage restrictions
|
||||
|
||||
p None. But this is a non-commercial and free service, so please be reasonable, and consider <a href="#donate">donating</a> if this service helped you out in some way!
|
||||
|
||||
h2 Can I use this from a browser?
|
||||
|
||||
p Yes. Cross-domain connections are allowed. But again, be reasonable - if you expect to be making hundreds or thousands of simultaneous connections, please run your own proxy.
|
||||
|
||||
a(name="donate")
|
||||
h2 Donate
|
||||
|
||||
p If you like this service, help keep it running by making a donation!
|
||||
|
||||
.donation-method.paypal
|
||||
h3 Donating once through PayPal
|
||||
|
||||
form(action="https://www.paypal.com/cgi-bin/webscr", method="post", target="_top")
|
||||
input(type="hidden", name="cmd", value="_donations")
|
||||
input(type="hidden", name="business", value="AQ9A6XVWUWHCC")
|
||||
input(type="hidden", name="lc", value="en")
|
||||
input(type="hidden", name="item_name", value="Pastebin Stream")
|
||||
input(type="hidden", name="item_number", value="pastebin-stream")
|
||||
input(type="hidden", name="currency_code", value="EUR")
|
||||
input(type="hidden", name="cn", value="Message:")
|
||||
input(type="hidden", name="no_note", value="0")
|
||||
input(type="hidden", name="no_shipping", value="1")
|
||||
input(type="hidden", name="rm", value="1")
|
||||
input(type="hidden", name="return", value="https://pastebin-stream.cryto.net/")
|
||||
input(type="hidden", name="cancel_return", value="https://pastebin-stream.cryto.net/")
|
||||
input(type="hidden", name="bn", value="PP-DonationsBF:btn_donateCC_LG.gif:NonHosted")
|
||||
|
||||
.amount
|
||||
label Amount:
|
||||
| €
|
||||
input(type="text", name="amount", value="10.00")
|
||||
|
||||
.submit
|
||||
input(type="image", src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif", border="0", name="submit", alt="PayPal, de veilige en complete manier van online betalen.")
|
||||
img(alt="", border=0, src="https://www.paypalobjects.com/en_EN/i/scr/pixel.gif", width="1", height="1")
|
||||
|
||||
.donation-method.paypal.monthly
|
||||
h3 Donating monthly through PayPal
|
||||
|
||||
form(action="https://www.paypal.com/cgi-bin/webscr", method="post", target="_top")
|
||||
input(type="hidden", name="cmd", value="_xclick-subscriptions")
|
||||
input(type="hidden", name="business", value="AQ9A6XVWUWHCC")
|
||||
input(type="hidden", name="lc", value="en")
|
||||
input(type="hidden", name="item_name", value="Pastebin Stream (Monthly Donation)")
|
||||
input(type="hidden", name="item_number", value="pastebin-stream-monthly")
|
||||
input(type="hidden", name="currency_code", value="EUR")
|
||||
input(type="hidden", name="cn", value="Message:")
|
||||
input(type="hidden", name="no_note", value="0")
|
||||
input(type="hidden", name="no_shipping", value="1")
|
||||
input(type="hidden", name="rm", value="1")
|
||||
input(type="hidden", name="p3", value="1")
|
||||
input(type="hidden", name="t3", value="M")
|
||||
input(type="hidden", name="return", value="https://pastebin-stream.cryto.net/")
|
||||
input(type="hidden", name="cancel_return", value="https://pastebin-stream.cryto.net/")
|
||||
input(type="hidden", name="bn", value="PP-DonationsBF:btn_donateCC_LG.gif:NonHosted")
|
||||
|
||||
.amount
|
||||
label Amount:
|
||||
| €
|
||||
input(type="text", name="a3", value="5.00")
|
||||
| /month
|
||||
|
||||
.submit
|
||||
input(type="image", src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif", border="0", name="submit", alt="PayPal, de veilige en complete manier van online betalen.")
|
||||
img(alt="", border=0, src="https://www.paypalobjects.com/en_EN/i/scr/pixel.gif", width="1", height="1")
|
||||
|
||||
.donation-method.bitcoin
|
||||
h3 Donating through Bitcoin
|
||||
|
||||
p Please send your donation to <strong>13quzALQ98dpJLPVcVA1rbEbE9Brf6VKyc</strong> :)
|
||||
|
||||
h2 Future plans
|
||||
|
||||
p In the future, I will likely add support for other Pastebin services as well. Suggestions are welcome - send me an e-mail!
|
||||
|
||||
h2 Contact
|
||||
|
||||
p If you need to reach me for some reason, you can e-mail me at <a href="admin@cryto.net">admin@cryto.net</a>.
|
||||
|
||||
h2 API documentation
|
||||
|
||||
p This API exposes a single endpoint, at <code>wss://pastebin-stream.cryto.net/stream</code>. Every command below is meant to be sent through that connection. The server expects well-formed JSON, and will kill a connection if it sends malformed data.
|
||||
|
||||
p No special clients are required; you can simply use the standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_client_applications">WebSocket API</a> in browsers. For Node.js, I recommend using <a href="https://www.npmjs.com/package/ws">the <code>ws</code> library</a>. For other languages, simply use whichever WebSocket implementation is available to you.
|
||||
|
||||
p The server sends out a 'ping' roughly every 5 seconds. If you wish to detect disconnections, this is what you'll want to check for.
|
||||
|
||||
h3 Making sure you don't miss anything
|
||||
|
||||
p The server maintains a backlog of pastes. The exact amount is subject to change, but assume "several hundred". You can make a backlog request to obtain these pastes from the backlog.
|
||||
|
||||
p When connecting for the first time, you should do the following to obtain the maximum amount of pastes and make sure you're not left with a gap:
|
||||
|
||||
ol
|
||||
li Set up a message handler.
|
||||
li Send a subscribe request to receive <code>newPaste</code> events (immediately storing pastes once they start coming in).
|
||||
li Request a backlog <em>after</em> sending the subscribe request (either full or partial).
|
||||
li Once the backlog is returned, remove any pastes that you already received through <code>newPaste</code> events, and then insert the backlog <em>before</em> the already-received pastes.
|
||||
|
||||
p The API also includes a <code>counter</code> property with every paste. This is an incrementing counter managed by the server, that can be used to obtain missed pastes after a reconnect.
|
||||
|
||||
p If you've detected a connection loss and initiated a new connection, you should do the following to obtain all the pastes you missed:
|
||||
|
||||
ol
|
||||
li Set up a message handler.
|
||||
li Send a subscribe request to receive <code>newPaste</code> events (immediately storing pastes once they start coming in).
|
||||
li Request a backlog <em>after</em> sending the subscribe request, <strong>specifying the <code>since</code> parameter</strong> - it should contain the <code>counter</code> of the last paste you've received on the previous connection.
|
||||
li Once the backlog is returned, remove any pastes that you already received through <code>newPaste</code> events, and then insert the backlog <em>before</em> the already-received pastes.
|
||||
|
||||
p Note that when the server restarts, <strong>the <code>counter</code> values will be reset to zero</strong>. This means that a given <code>counter</code> value is only unique until the next server restart.
|
||||
|
||||
h3 Paste format
|
||||
|
||||
p A paste object looks like the following:
|
||||
|
||||
pre.
|
||||
{
|
||||
"counter": 521,
|
||||
"service": "pastebinCom",
|
||||
"id": "V2rpgLwN",
|
||||
"title": "Paste Title Goes Here",
|
||||
"date": 1489876199,
|
||||
"expiry": 0,
|
||||
"language": "text",
|
||||
"url": "http://pastebin.com/gPifdLrb"
|
||||
"contents": "[ ... paste contents go here ... ]"
|
||||
}
|
||||
|
||||
p An explanation of each of the properties:
|
||||
|
||||
ul
|
||||
li <strong>counter</strong>: Unique incrementing counter. Used for catching up on missed pastes.
|
||||
li <strong>service</strong>: The pastebin service that this paste originated from. Currently, <code>pastebinCom</code> is the only option here.
|
||||
li <strong>id</strong>: The identifier of the paste, as specified by the pastebin service. The meaning of this is service-specific, and there is no guarantee that it will be unique between services.
|
||||
li <strong>title</strong>: <em>Optional.</em> The title given to the paste.
|
||||
li <strong>date</strong>: <em>Optional.</em> A UNIX timestamp (in seconds) indicating when the paste was originally posted.
|
||||
li <strong>expiry</strong>: <em>Optional.</em> The time <em>in seconds</em> after which the paste will expire (relative to the original posting date).
|
||||
li <strong>language</strong>: <em>Optional.</em> The language that the user picked for this paste. The values are service-specific.
|
||||
li <strong>username</strong>: <em>Optional.</em> The username of the user that posted the paste.
|
||||
li <strong>url</strong>: The original URL for viewing the paste.
|
||||
li <strong>contents</strong>: The raw contents of the paste, as a string.
|
||||
|
||||
p Any <em>optional</em> properties may be missing. You should take this into account in your implementation.
|
||||
|
||||
|
||||
h3 Subscribing to pastes
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "subscribe"
|
||||
}
|
||||
|
||||
p You should send this immediately after connecting. Once you do, you will start receiving pastes. Incoming pastes look like the following:
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "newPaste",
|
||||
"data": <a single paste object goes here>
|
||||
}
|
||||
|
||||
h3 Requesting a backlog
|
||||
|
||||
p There are a few options for request a backlog, but the <em>response</em> is always in the same format:
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "backlog",
|
||||
"results": [
|
||||
... an array of paste objects goes here ...
|
||||
]
|
||||
}
|
||||
|
||||
h4 Requesting the full backlog
|
||||
|
||||
p This retrieves every single paste that the server has in its backlog. Note that the response may be large.
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "backlog",
|
||||
"all": true
|
||||
}
|
||||
|
||||
h4 Requesting the last X entries
|
||||
|
||||
p This retrieves only the last X entries in the backlog, where X is an amount you specify.
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "backlog",
|
||||
"last": 25
|
||||
}
|
||||
|
||||
h4 Requesting the backlog since a specific paste
|
||||
|
||||
p This retrieves every paste in the backlog since a given <code>counter</code> value.
|
||||
|
||||
pre.
|
||||
{
|
||||
"type": "backlog",
|
||||
"since": 51291
|
||||
}
|
||||
Loading…
Reference in New Issue