Fix LCSC scraper
parent
795ed9c760
commit
cbb1e11fb2
@ -1,19 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
const Promise = require("bluebird");
|
||||
|
||||
const assureResponse = require("../shared/assure-response");
|
||||
const matchOrFail = require("../shared/match-or-fail");
|
||||
|
||||
module.exports = function ({ session }) {
|
||||
return function getCSRFToken() {
|
||||
return Promise.try(() => {
|
||||
return session.get("https://lcsc.com/products");
|
||||
}).then((response) => {
|
||||
assureResponse(response);
|
||||
|
||||
let [ extractedToken ] = matchOrFail(/'X-CSRF-TOKEN': '([^']+)'/, response.body);
|
||||
return extractedToken;
|
||||
});
|
||||
};
|
||||
};
|
@ -1,33 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
|
||||
// FIXME: This is a bit of a hack to persist the CSRF token across calls. There is probably a better solution for this, but that sort of state management needs to be handled on a scraping-server level, probably.
|
||||
let token;
|
||||
|
||||
module.exports = function (state) {
|
||||
const getCsrfToken = require("./get-csrf-token")(state);
|
||||
|
||||
return async function withCSRFToken(callback) {
|
||||
async function obtainToken() {
|
||||
token = await getCsrfToken();
|
||||
}
|
||||
|
||||
async function attemptCallback() {
|
||||
let response = await callback(token);
|
||||
|
||||
if (response.statusCode === 419) {
|
||||
await obtainToken();
|
||||
|
||||
return attemptCallback();
|
||||
} else {
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
if (token == null) {
|
||||
await obtainToken();
|
||||
}
|
||||
|
||||
return attemptCallback();
|
||||
};
|
||||
};
|
Loading…
Reference in New Issue