Re-queue on 5xx status codes for paste fetches

master
Sven Slootweg 7 years ago
parent bed9fefc9c
commit ecb3b51bec

@ -35,8 +35,12 @@ module.exports = function createPastebinComScraper(options = {}) {
return bhttp.get(`https://pastebin.com/api_scrape_item.php?i=${task.pasteKey}`); return bhttp.get(`https://pastebin.com/api_scrape_item.php?i=${task.pasteKey}`);
}).then((response) => { }).then((response) => {
if (response.statusCode !== 200) { if (response.statusCode !== 200) {
// FIXME: Retry! if (response.statusCode >= 500 && response.statusCode < 600) {
// FIXME: The below logic could lead to an infinite recursion if Pastebin never comes back up
return queue.push("fetchPaste", task);
} else {
throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while retrieving a raw paste: ${response.statusCode}`, {type: "statusCode"}); throw new errors.HttpError(`Encountered a non-200 status code for Pastebin.com while retrieving a raw paste: ${response.statusCode}`, {type: "statusCode"});
}
} else if (response.body.toString().indexOf("Please slow down, you are hitting our servers unnecessarily hard!") === 0) { } else if (response.body.toString().indexOf("Please slow down, you are hitting our servers unnecessarily hard!") === 0) {
throw new errors.HttpError("Got rate-limited!", {type: "rateLimited"}); throw new errors.HttpError("Got rate-limited!", {type: "rateLimited"});
} else { } else {

Loading…
Cancel
Save