joepie91
/
pastebin-stream
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Initial commit

Browse Source
master
Sven Slootweg 8 years ago
commit 8a9864ea81
21 changed files with 27124 additions and 0 deletions
Show Stats Download Patch File Download Diff File

3
.gitignore vendored
Unescape Escape View File

@ -0,0 +1,3 @@
errors
node_modules
config.json

9
config.json.example
Unescape Escape View File

@ -0,0 +1,9 @@
{
"scraperSettings": {
"pastebinCom": {
"listInterval": 60,
"listLimit": 100,
"pasteInterval": 1
}
}
}

116
gulpfile.js
Unescape Escape View File

@ -0,0 +1,116 @@
const Promise = require("bluebird");
const path = require("path");
const gulp = require("gulp");
const webpackStream = require("webpack-stream");
const webpack = require("webpack");
const gulpNamedLog = require("gulp-named-log");
const gulpRename = require("gulp-rename");
const gulpNodemon = require("gulp-nodemon");
const gulpCached = require("gulp-cached");
const presetSCSS = require("@joepie91/gulp-preset-scss");
const awaitServer = require("await-server");
const gulpLivereload = require("gulp-livereload");
const patchLivereloadLogger = require("@joepie91/gulp-partial-patch-livereload-logger");
patchLivereloadLogger(gulpLivereload);
let config = {
scss: {
source: "./src/scss/**/*.scss",
destination: "./public/css/"
},
cssModules: []
}
let serverLogger = gulpNamedLog("server");
gulp.task("nodemon", ["scss", "livereload"], () => {
gulpNodemon({
script: "server.js",
ignore: [
"gulpfile.js",
"node_modules",
"public"
],
ext: "js pug"
}).on("start", () => {
Promise.try(() => {
serverLogger.info("Starting...");
return awaitServer(3000);
}).then(() => {
serverLogger.info("Started!");
gulpLivereload.changed("*");
});
});
});
gulp.task("scss", () => {
return gulp.src(config.scss.source)
.pipe(presetSCSS({
livereload: gulpLivereload
}))
.pipe(gulp.dest(config.scss.destination));
});
gulp.task("livereload", () => {
gulpLivereload.listen({
quiet: true
});
});
gulp.task("webpack-watch", ["livereload"], () => {
return gulp.src("./src/frontend/index.js")
.pipe(webpackStream({
watch: true,
module: {
preLoaders: [{
test: /\.tag$/,
loader: "riotjs-loader",
exclude: /node_modules/,
query: {
type: "es6",
template: "pug",
parserOptions: {
js: {
presets: ["es2015-riot"]
}
}
}
}],
loaders: [{
test: /\.js/,
loader: "babel-loader",
query: {
presets: ["es2015"]
}
}, {
test: /\.json$/,
loader: "json-loader"
}]
},
resolve: {
extensions: [
"",
".tag",
".web.js", ".js",
".web.json", ".json"
]
},
plugins: [ new webpack.ProvidePlugin({riot: "riot"}) ],
debug: true
}))
.pipe(gulpRename("bundle.js"))
.pipe(gulpNamedLog("webpack").stream())
.pipe(gulpLivereload())
.pipe(gulp.dest("./public/js/"));
});
gulp.task("watch-css", () => {
gulp.watch(config.scss.source, ["scss"]);
});
gulp.task("watch", ["webpack-watch", "nodemon", "watch-css"]);
gulp.task("default", ["watch"]);

53
package.json
Unescape Escape View File

@ -0,0 +1,53 @@
{
"name": "pastebin-stream",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git@git.cryto.net:joepie91/pastebin-stream.git"
},
"keywords": [],
"author": "Sven Slootweg",
"license": "WTFPL",
"dependencies": {
"bhttp": "^1.2.4",
"bluebird": "^3.5.0",
"create-error": "^0.3.1",
"create-event-emitter": "^1.0.0",
"debug": "^2.6.3",
"default-value": "^1.0.0",
"document-ready-promise": "^3.0.1",
"express": "^4.15.2",
"express-ws": "^3.0.0",
"mkdirp": "^0.5.1",
"moment": "^2.18.0",
"promise-task-queue": "^1.2.0",
"pug": "^2.0.0-beta11",
"report-errors": "^1.0.0",
"ws": "^2.2.1"
},
"devDependencies": {
"@joepie91/gulp-partial-patch-livereload-logger": "^1.0.1",
"@joepie91/gulp-preset-scss": "^1.0.3",
"babel-core": "^6.22.1",
"babel-loader": "^6.2.10",
"babel-preset-es2015": "^6.22.0",
"babel-preset-es2015-riot": "^1.1.0",
"document-ready-promise": "^3.0.1",
"gulp": "^3.9.1",
"gulp-cached": "^1.1.1",
"gulp-livereload": "^3.8.1",
"gulp-named-log": "^1.0.1",
"gulp-nodemon": "^2.2.1",
"gulp-rename": "^1.2.2",
"riot": "^3.3.2",
"riotjs-loader": "^4.0.0",
"webpack": "^1.14.0",
"webpack-stream": "^3.2.0",
"wsurl": "^1.0.0"
}
}

59
public/css/style.css
Unescape Escape View File

@ -0,0 +1,59 @@
body {
font-family: sans-serif;
line-height: 1.4; }
.wrapper {
max-width: 900px;
margin: 0px auto; }
pre {
padding: 16px;
border: 1px solid #bababa;
background-color: #eeeeee;
border-radius: 5px; }
code {
padding: 3px 7px 3px 7px; }
code, pre {
font-size: 16px;
background-color: #eeeeee;
border-radius: 5px; }
.donation-method {
padding: 16px;
border: 1px solid #bababa;
background-color: #eeeeee;
border-radius: 5px;
margin-bottom: 8px; }
.donation-method h3 {
margin-top: 0px;
margin-bottom: 7px;
border-bottom: none; }
.donation-method p {
margin: 5px 0px; }
.donation-method .amount {
font-size: 16px; }
.donation-method .amount label {
margin-right: 7px; }
.donation-method .amount input {
margin: 0px 2px;
font-size: 16px;
padding: 4px 5px;
border-radius: 3px;
border: 1px solid silver;
width: 80px; }
.donation-method.monthly .amount input {
width: 55px; }
.donation-method .amount, .donation-method .submit {
display: inline; }
.donation-method .submit {
position: relative;
top: 8px;
left: 12px; }
h2 {
border-bottom: 1px solid silver; }
h3 {
border-bottom: 1px solid #cdcdcd; }

26001
public/js/bundle.js
View File

File diff suppressed because it is too large Load Diff

81
server.js
Unescape Escape View File

@ -0,0 +1,81 @@
'use strict';
const errorReporter = require("./src/error-reporting");
const path = require("path");
const express = require("express");
const expressWs = require("express-ws");
const config = require("./config.json");
const clientStore = require("./src/client-store")();
const scraper = require("./src/scraper")({
scrapePastebinCom: true,
scraperSettings: config.scraperSettings
});
scraper.on("newPaste", (paste) => {
clientStore.broadcast({
type: "newPaste",
data: paste
});
});
scraper.on("warning", (error) => {
console.error(error);
});
scraper.on("error", (error) => {
errorReporter.report(error, {
source: "scraper"
});
});
clientStore.on("error", (error) => {
errorReporter.report(error, {
source: "clientStore"
});
});
let app = express();
expressWs(app);
app.set("views", path.join(__dirname, "views"));
app.set("view engine", "pug");
app.use(express.static(path.join(__dirname, "public")));
app.get("/", (req, res) => {
res.render("index");
});
app.ws("/stream", (ws, req) => {
let client = clientStore.add(ws);
client.on("message", (message) => {
if (message.type === "backlog") {
if (message.all === true) {
client.send({
type: "backlog",
results: scraper.pasteStore.all(message.since)
});
} else if (typeof message.since === "number") {
client.send({
type: "backlog",
results: scraper.pasteStore.since(message.since)
});
} else if (typeof message.last === "number") {
client.send({
type: "backlog",
results: scraper.pasteStore.last(message.last)
});
} else {
client.kill();
}
}
})
});
app.listen(3000, () => {
console.log("Listening on port 3000...");
});

9
src/clamp-zero.js
Unescape Escape View File

@ -0,0 +1,9 @@
'use strict';
module.exports = function clampZero(value) {
if (value < 0) {
return 0;
} else {
return value;
}
};

102
src/client-store.js
Unescape Escape View File

@ -0,0 +1,102 @@
'use strict';
const createEventEmitter = require("create-event-emitter");
const defaultValue = require("default-value");
function removeFromArray(array, item) {
let index = array.indexOf(item);
if (index !== -1) {
array.splice(index, 1);
}
}
module.exports = function createClientStore(options = {}) {
let clients = [];
let subscribedClients = [];
let store = createEventEmitter({
add: function addClient(client) {
let clientObject = createEventEmitter({
socket: client,
subscribed: false,
disconnected: false,
kill: function killClient() {
store.kill(client);
},
send: function sendMessage(message) {
let data = JSON.stringify(message);
this.socket.send(data, (error) => {
if (error != null) {
store.emit("error", error);
}
});
}
});
client.on("message", (data) => {
let message;
try {
message = JSON.parse(data);
} catch (err) {
/* User sent invalid data; disconnect them. */
return this.killClient(client);
}
if (message == null || typeof message !== "object") {
/* User sent invalid data; disconnect them. */
return this.killClient(client);
}
if (message.type === "subscribe") {
if (clientObject.subscribed === false) {
subscribedClients.push(clientObject);
clientObject.subscribed = true;
clientObject.emit("subscribed");
}
} else {
clientObject.emit("message", message);
}
});
client.on("close", (code, reason) => {
client.disconnected = true;
this.remove(clientObject);
});
clients.push(clientObject);
return clientObject;
},
kill: function killClient(client) {
client.close();
},
remove: function removeClient(client) {
removeFromArray(clients, client);
removeFromArray(subscribedClients, client);
},
broadcast: function broadcastMessage(message, clients) {
defaultValue(clients, subscribedClients).forEach((client) => {
client.send(message);
});
},
broadcastAll: function broadcastMessageToAll(message) {
this.broadcast(message, clients);
}
});
let pingInterval = defaultValue(options.pingInterval, 5000);
function pingClients() {
store.broadcastAll({
type: "ping"
});
setTimeout(pingClients, pingInterval);
}
pingClients();
return store;
};

20
src/error-reporting.js
Unescape Escape View File

@ -0,0 +1,20 @@
'use strict';
const path = require("path");
const reportErrors = require("report-errors");
const mkdirp = require("mkdirp");
let errorPath = path.join(__dirname, "../errors");
mkdirp.sync(errorPath);
module.exports = reportErrors(errorPath, {
doNotCrash: true,
handler: (error, context) => {
console.error("Unhandled error", error.stack);
if (error.rateLimited !== true) {
process.exit(1);
}
}
});

7
src/errors.js
Unescape Escape View File

@ -0,0 +1,7 @@
'use strict';
const createError = require("create-error");
module.exports = {
HttpError: createError("HttpError")
};

91
src/frontend/components/app.tag
Unescape Escape View File

@ -0,0 +1,91 @@
app
div.pasteBox
div.paste(each="{ paste in pastes }")
span.title(if="{ paste.title != null }") {paste.title}
span.title(if="{ paste.title == null }") (no title)
span.date {paste.formattedDate}
span.url {paste.url}
script.
const moment = require("moment");
const wsEndpoint = require("../ws-endpoint");
const pruneArray = require("../../prune-array");
function formatDate(date) {
return moment.unix(date).format("HH:mm:ss");
}
function addFormattedDate(paste) {
return Object.assign({
formattedDate: formatDate(paste.date)
}, paste);
}
this.pastes = [];
this.on("mount", () => {
let endpoint = wsEndpoint("/stream");
let socket = new WebSocket(endpoint);
socket.onmessage = (event) => {
let message = JSON.parse(event.data);
if (message.type === "newPaste") {
let newPaste = message.data;
this.pastes.push(addFormattedDate(newPaste));
} else if (message.type === "backlog") {
let unknownPastes = message.results.filter((paste) => {
return !this.pastes.some((existingPaste) => (existingPaste.id === paste.id))
}).map((paste) => {
return addFormattedDate(paste);
});
this.pastes = unknownPastes.concat(this.pastes);
}
pruneArray(this.pastes, 10);
this.update();
}
socket.onopen = (event) => {
socket.send(JSON.stringify({
type: "backlog",
last: 10
}));
socket.send(JSON.stringify({
type: "subscribe"
}));
}
this.update();
});
style(type="scss").
.pasteBox {
height: 310px;
border: 1px solid rgb(173, 173, 173);
overflow: auto;
.paste {
padding: 4px;
background-color: rgb(209, 209, 209);
border-bottom: 1px solid rgb(173, 173, 173);
.url {
color: gray;
}
.title {
font-weight: bold;
}
.title:after, .date:after {
content: "-";
margin: 0px 4px;
}
}
}

12
src/frontend/index.js
Unescape Escape View File

@ -0,0 +1,12 @@
'use strict';
const Promise = require("bluebird");
const documentReadyPromise = require("document-ready-promise");
const app = require("./components/app");
const riot = require("riot");
Promise.try(() => {
return documentReadyPromise();
}).then((result) => {
riot.mount("*");
});

8
src/frontend/ws-endpoint.js
Unescape Escape View File

@ -0,0 +1,8 @@
'use strict';
const wsurl = require("wsurl");
const url = require("url");
module.exports = function wsEndpoint(path) {
return wsurl(url.resolve(document.location.href, path));
}

37
src/paste-store.js
Unescape Escape View File

@ -0,0 +1,37 @@
'use strict';
const defaultValue = require("default-value");
const createEventEmitter = require("create-event-emitter");
const pruneArray = require("./prune-array");
const clampZero = require("./clamp-zero");
module.exports = function createPasteStore(options = {}) {
let pastes = [];
let counter = 0;
let pasteLimit = defaultValue(options.pasteLimit, 250);
return createEventEmitter({
add: function addPaste(pasteData) {
let paste = Object.assign({
counter: counter++
}, pasteData);
pastes.push(paste);
this.emit("added", paste);
pruneArray(pastes, pasteLimit);
},
since: function getPastesSince(counter) {
return pastes.filter((paste) => {
return (paste.counter > counter);
});
},
all: function getAllPastes() {
return pastes;
},
last: function getLastPastes(count) {
return pastes.slice(clampZero(pastes.length - count));
}
});
};

49
src/promise-set-interval.js
Unescape Escape View File

@ -0,0 +1,49 @@
'use strict';
const Promise = require("bluebird");
const createEventEmitter = require("create-event-emitter");
const clampZero = require("./clamp-zero");
module.exports = function promiseSetInterval(callback, interval, options = {}) {
let lastInvocation = Date.now();
let timeout;
let emitter = createEventEmitter({
cancel: function cancelInterval() {
clearTimeout(timeout);
}
});
function scheduleNext() {
let delta = Date.now() - lastInvocation;
let timeout = clampZero(interval - delta);
scheduleInvocation(timeout);
}
function scheduleInvocation(nextTimeout) {
timeout = setTimeout(() => {
Promise.try(() => {
lastInvocation = Date.now();
return callback();
}).then(() => {
scheduleNext();
}).catch((err) => {
emitter.emit("error", err);
if (!options.abortOnError) {
scheduleNext();
}
});
}, nextTimeout);
}
if (options.startImmediately) {
scheduleInvocation(0);
} else {
scheduleNext();
}
return emitter;
};

9
src/prune-array.js
Unescape Escape View File

@ -0,0 +1,9 @@
'use strict';
/* CAUTION: This mutates the array! */
module.exports = function pruneArray(array, maxItems) {
if (array.length > maxItems) {
array.splice(0, array.length - maxItems);
}
};

57
src/scraper.js
Unescape Escape View File

@ -0,0 +1,57 @@
'use strict';
const createEventEmitter = require("create-event-emitter");
const defaultValue = require("default-value");
function ifNotEmpty(value) {
if (value !== "") {
return value;
}
}
module.exports = function createScraper(options = {}) {
let scraperSettings = defaultValue(options.scraperSettings, {});
const pasteStore = require("./paste-store")();
const pastebinComScraper = require("./scrapers/pastebin-com")(scraperSettings.pastebinCom);
const errors = require("./errors");
let scraper = createEventEmitter({
pasteStore: pasteStore
});
pastebinComScraper.on("paste", (paste) => {
let postedDate = parseInt(paste.date);
let expiryDate = parseInt(paste.expire);
pasteStore.add({
site: "pastebinCom",
url: paste.full_url,
date: postedDate,
id: paste.key,
expire: (expiryDate !== 0) ? (expiryDate - postedDate) : undefined,
username: ifNotEmpty(paste.user),
language: paste.syntax,
title: ifNotEmpty(paste.title),
contents: paste.raw
});
});
pastebinComScraper.on("error", (error) => {
if (error instanceof errors.HttpError && error.rateLimited !== true) {
scraper.emit("warning", error);
} else {
scraper.emit("error", error);
}
});
if (options.scrapePastebinCom) {
pastebinComScraper.start();
}
pasteStore.on("added", (paste) => {
scraper.emit("newPaste", paste);
});
return scraper;
};

93
src/scrapers/pastebin-com.js
Unescape Escape View File

@ -0,0 +1,93 @@
'use strict';
const Promise = require("bluebird");
const bhttp = require("bhttp");
const promiseTaskQueue = require("promise-task-queue");
const createEventEmitter = require("create-event-emitter");
const defaultValue = require("default-value");
const debug = require("debug")("pastebinStream:scrapers:pastebinCom");
const promiseSetInterval = require("../promise-set-interval");
const errors = require("../errors");
function tryParseBody(body) {
try {
return JSON.parse(body);
} catch (err) {
throw new errors.HttpError(`Got rate-limited? Error message: ${body}`, {type: "rateLimited"});
}
}
module.exports = function createPastebinComScraper(options = {}) {
let queue = promiseTaskQueue();
let knownPastes = [];
let previousKnownPastes = [];
queue.define("fetchPaste", (task) => {
return Promise.try(() => {
return bhttp.get(`http://pastebin.com/api_scrape_item.php?i=${task.pasteKey}`);
}).then((response) => {
if (response.statusCode !== 200) {
// FIXME: Retry!
throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while retrieving a raw paste: ${response.statusCode}`, {type: "statusCode"});
} else if (response.body.toString().indexOf("Please slow down, you are hitting our servers unnecessarily hard!") === 0) {
throw new errors.HttpError("Got rate-limited!", {type: "rateLimited"});
} else {
return response.body.toString();
}
})
}, {
interval: options.pasteInterval
});
let loop;
return createEventEmitter({
stop: function stopScraper() {
if (loop != null) {
this.emit("stopped");
loop.cancel();
}
},
start: function startScraper() {
loop = promiseSetInterval(() => {
return Promise.try(() => {
return bhttp.get(`http://pastebin.com/api_scraping.php?limit=${defaultValue(options.listLimit, 100)}`, {
noDecode: true /* Because Pastebin.com errors aren't JSON... */
});
}).then((response) => {
if (response.statusCode !== 200) {
throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while listing the most recent pastes: ${response.statusCode}`, {type: "statusCode"});
} else {
return tryParseBody(response.body).reverse();
}
}).tap((pastes) => {
previousKnownPastes = knownPastes;
knownPastes = pastes.map(paste => paste.key);
}).filter((paste) => {
return (!previousKnownPastes.includes(paste.key));
}).tap((pastes) => {
debug(`Found ${pastes.length} new pastes`);
}).each((paste) => {
/* We *intentionally* do not return the Promise chain below; we don't want to block the interval with queue items. */
Promise.try(() => {
return queue.push("fetchPaste", {
pasteKey: paste.key
});
}).then((rawPaste) => {
this.emit("paste", Object.assign({
raw: rawPaste
}, paste));
}).catch((err) => {
this.emit("error", err);
});
}).catch((err) => {
/* This is where eg. rate-limiting errors will end up. */
this.emit("error", err);
});
}, defaultValue(options.listInterval, 60) * 1000, {
startImmediately: true
});
}
});
};

83
src/scss/style.scss
Unescape Escape View File

@ -0,0 +1,83 @@
body {
font-family: sans-serif;
line-height: 1.4;
}
.wrapper {
max-width: 900px;
margin: 0px auto;
}
pre {
padding: 16px;
border: 1px solid rgb(186, 186, 186);
background-color: rgb(238, 238, 238);
border-radius: 5px;
}
code {
padding: 3px 7px 3px 7px;
}
code, pre {
font-size: 16px;
background-color: #eeeeee;
border-radius: 5px;
}
.donation-method {
padding: 16px;
border: 1px solid rgb(186, 186, 186);
background-color: rgb(238, 238, 238);
border-radius: 5px;
margin-bottom: 8px;
h3 {
margin-top: 0px;
margin-bottom: 7px;
border-bottom: none;
}
p {
margin: 5px 0px;
}
.amount {
font-size: 16px;
label {
margin-right: 7px;
}
input {
margin: 0px 2px;
font-size: 16px;
padding: 4px 5px;
border-radius: 3px;
border: 1px solid silver;
width: 80px;
}
}
&.monthly .amount input {
width: 55px;
}
.amount, .submit {
display: inline;
}
.submit {
position: relative;
top: 8px;
left: 12px;
}
}
h2 {
border-bottom: 1px solid silver;
}
h3 {
border-bottom: 1px solid rgb(205, 205, 205);
}

225
views/index.pug
Unescape Escape View File

@ -0,0 +1,225 @@
doctype html
html
head
title Pastebin Stream
script(src="/js/bundle.js")
link(rel="stylesheet", href="/css/style.css")
body
.wrapper
h1 Pastebin Stream
p This is a simple streaming (WebSocket) API, providing a live feed of (public) pastes as they are submitted to <a href="http://pastebin.com/">Pastebin.com</a>. You can find the source code <a href="https://git.cryto.net/joepie91/pastebin-stream">here</a>.
p Since Pastebin does not offer a streaming API itself, this API works by regularly polling the Pastebin API for new posts. This means that posts may be delayed by about a minute, and that they'll be broadcast in 'batches'.
h2 Preview
p The latest pastes:
app
h2 Usage restrictions
p None. But this is a non-commercial and free service, so please be reasonable, and consider <a href="#donate">donating</a> if this service helped you out in some way!
h2 Can I use this from a browser?
p Yes. Cross-domain connections are allowed. But again, be reasonable - if you expect to be making hundreds or thousands of simultaneous connections, please run your own proxy.
a(name="donate")
h2 Donate
p If you like this service, help keep it running by making a donation!
.donation-method.paypal
h3 Donating once through PayPal
form(action="https://www.paypal.com/cgi-bin/webscr", method="post", target="_top")
input(type="hidden", name="cmd", value="_donations")
input(type="hidden", name="business", value="AQ9A6XVWUWHCC")
input(type="hidden", name="lc", value="en")
input(type="hidden", name="item_name", value="Pastebin Stream")
input(type="hidden", name="item_number", value="pastebin-stream")
input(type="hidden", name="currency_code", value="EUR")
input(type="hidden", name="cn", value="Message:")
input(type="hidden", name="no_note", value="0")
input(type="hidden", name="no_shipping", value="1")
input(type="hidden", name="rm", value="1")
input(type="hidden", name="return", value="https://pastebin-stream.cryto.net/")
input(type="hidden", name="cancel_return", value="https://pastebin-stream.cryto.net/")
input(type="hidden", name="bn", value="PP-DonationsBF:btn_donateCC_LG.gif:NonHosted")
.amount
label Amount:
| €
input(type="text", name="amount", value="10.00")
.submit
input(type="image", src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif", border="0", name="submit", alt="PayPal, de veilige en complete manier van online betalen.")
img(alt="", border=0, src="https://www.paypalobjects.com/en_EN/i/scr/pixel.gif", width="1", height="1")
.donation-method.paypal.monthly
h3 Donating monthly through PayPal
form(action="https://www.paypal.com/cgi-bin/webscr", method="post", target="_top")
input(type="hidden", name="cmd", value="_xclick-subscriptions")
input(type="hidden", name="business", value="AQ9A6XVWUWHCC")
input(type="hidden", name="lc", value="en")
input(type="hidden", name="item_name", value="Pastebin Stream (Monthly Donation)")
input(type="hidden", name="item_number", value="pastebin-stream-monthly")
input(type="hidden", name="currency_code", value="EUR")
input(type="hidden", name="cn", value="Message:")
input(type="hidden", name="no_note", value="0")
input(type="hidden", name="no_shipping", value="1")
input(type="hidden", name="rm", value="1")
input(type="hidden", name="p3", value="1")
input(type="hidden", name="t3", value="M")
input(type="hidden", name="return", value="https://pastebin-stream.cryto.net/")
input(type="hidden", name="cancel_return", value="https://pastebin-stream.cryto.net/")
input(type="hidden", name="bn", value="PP-DonationsBF:btn_donateCC_LG.gif:NonHosted")
.amount
label Amount:
| €
input(type="text", name="a3", value="5.00")
| /month
.submit
input(type="image", src="https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif", border="0", name="submit", alt="PayPal, de veilige en complete manier van online betalen.")
img(alt="", border=0, src="https://www.paypalobjects.com/en_EN/i/scr/pixel.gif", width="1", height="1")
.donation-method.bitcoin
h3 Donating through Bitcoin
p Please send your donation to <strong>13quzALQ98dpJLPVcVA1rbEbE9Brf6VKyc</strong> :)
h2 Future plans
p In the future, I will likely add support for other Pastebin services as well. Suggestions are welcome - send me an e-mail!
h2 Contact
p If you need to reach me for some reason, you can e-mail me at <a href="admin@cryto.net">admin@cryto.net</a>.
h2 API documentation
p This API exposes a single endpoint, at <code>wss://pastebin-stream.cryto.net/stream</code>. Every command below is meant to be sent through that connection. The server expects well-formed JSON, and will kill a connection if it sends malformed data.
p No special clients are required; you can simply use the standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_client_applications">WebSocket API</a> in browsers. For Node.js, I recommend using <a href="https://www.npmjs.com/package/ws">the <code>ws</code> library</a>. For other languages, simply use whichever WebSocket implementation is available to you.
p The server sends out a 'ping' roughly every 5 seconds. If you wish to detect disconnections, this is what you'll want to check for.
h3 Making sure you don't miss anything
p The server maintains a backlog of pastes. The exact amount is subject to change, but assume "several hundred". You can make a backlog request to obtain these pastes from the backlog.
p When connecting for the first time, you should do the following to obtain the maximum amount of pastes and make sure you're not left with a gap:
ol
li Set up a message handler.
li Send a subscribe request to receive <code>newPaste</code> events (immediately storing pastes once they start coming in).
li Request a backlog <em>after</em> sending the subscribe request (either full or partial).
li Once the backlog is returned, remove any pastes that you already received through <code>newPaste</code> events, and then insert the backlog <em>before</em> the already-received pastes.
p The API also includes a <code>counter</code> property with every paste. This is an incrementing counter managed by the server, that can be used to obtain missed pastes after a reconnect.
p If you've detected a connection loss and initiated a new connection, you should do the following to obtain all the pastes you missed:
ol
li Set up a message handler.
li Send a subscribe request to receive <code>newPaste</code> events (immediately storing pastes once they start coming in).
li Request a backlog <em>after</em> sending the subscribe request, <strong>specifying the <code>since</code> parameter</strong> - it should contain the <code>counter</code> of the last paste you've received on the previous connection.
li Once the backlog is returned, remove any pastes that you already received through <code>newPaste</code> events, and then insert the backlog <em>before</em> the already-received pastes.
p Note that when the server restarts, <strong>the <code>counter</code> values will be reset to zero</strong>. This means that a given <code>counter</code> value is only unique until the next server restart.
h3 Paste format
p A paste object looks like the following:
pre.
{
"counter": 521,
"service": "pastebinCom",
"id": "V2rpgLwN",
"title": "Paste Title Goes Here",
"date": 1489876199,
"expiry": 0,
"language": "text",
"url": "http://pastebin.com/gPifdLrb"
"contents": "[ ... paste contents go here ... ]"
}
p An explanation of each of the properties:
ul
li <strong>counter</strong>: Unique incrementing counter. Used for catching up on missed pastes.
li <strong>service</strong>: The pastebin service that this paste originated from. Currently, <code>pastebinCom</code> is the only option here.
li <strong>id</strong>: The identifier of the paste, as specified by the pastebin service. The meaning of this is service-specific, and there is no guarantee that it will be unique between services.
li <strong>title</strong>: <em>Optional.</em> The title given to the paste.
li <strong>date</strong>: <em>Optional.</em> A UNIX timestamp (in seconds) indicating when the paste was originally posted.
li <strong>expiry</strong>: <em>Optional.</em> The time <em>in seconds</em> after which the paste will expire (relative to the original posting date).
li <strong>language</strong>: <em>Optional.</em> The language that the user picked for this paste. The values are service-specific.
li <strong>username</strong>: <em>Optional.</em> The username of the user that posted the paste.
li <strong>url</strong>: The original URL for viewing the paste.
li <strong>contents</strong>: The raw contents of the paste, as a string.
p Any <em>optional</em> properties may be missing. You should take this into account in your implementation.
h3 Subscribing to pastes
pre.
{
"type": "subscribe"
}
p You should send this immediately after connecting. Once you do, you will start receiving pastes. Incoming pastes look like the following:
pre.
{
"type": "newPaste",
"data": &lt;a single paste object goes here&gt;
}
h3 Requesting a backlog
p There are a few options for request a backlog, but the <em>response</em> is always in the same format:
pre.
{
"type": "backlog",
"results": [
... an array of paste objects goes here ...
]
}
h4 Requesting the full backlog
p This retrieves every single paste that the server has in its backlog. Note that the response may be large.
pre.
{
"type": "backlog",
"all": true
}
h4 Requesting the last X entries
p This retrieves only the last X entries in the backlog, where X is an amount you specify.
pre.
{
"type": "backlog",
"last": 25
}
h4 Requesting the backlog since a specific paste
p This retrieves every paste in the backlog since a given <code>counter</code> value.
pre.
{
"type": "backlog",
"since": 51291
}
Write Preview
Loading…
Cancel
Save