Sven Slootweg 4 months ago
parent
commit
78a0c5a051
  1. 13
      migrations/20211215172404_store-date.js
  2. 20
      public/css/style.css
  3. 22
      src/app.js
  4. 20
      src/css/style.css
  5. 2
      src/frontend/components/datasheet-search.jsx
  6. 64
      src/sync/index.js
  7. 8
      src/sync/update-stream.js
  8. 43
      src/views/_layout.jsx
  9. 33
      src/views/contact.jsx
  10. 45
      src/views/datasheets/index.jsx
  11. 12
      yarn.lock

13
migrations/20211215172404_store-date.js

@ -0,0 +1,13 @@
"use strict";
module.exports.up = function(knex, Promise) {
return knex.schema.table("datasheets_products", (table) => {
table.timestamp("last_updated").index().notNull();
});
};
module.exports.down = function(knex, Promise) {
return knex.schema.table("datasheets_products", (table) => {
table.dropColumn("last_updated");
});
};

20
public/css/style.css

@ -60,9 +60,29 @@ html, body {
font-size: 1.3em;
}
.logoContainer .betaTag {
width: 1px; /* Out-of-box alignment hack */
position: absolute;
right: -.3em;
bottom: 0;
font-style: italic;
color: rgb(218, 13, 13);
font-size: 1.3em;
}
.counter {
margin-bottom: .5em;
font-style: italic;
font-size: .9em;
text-align: right;
}
.staticContent {
margin: 0 2em;
max-width: 900px;
}
.linkSpacer {
margin: 0 .5em;
}

22
src/app.js

@ -18,12 +18,26 @@ createSynchronizer("datasheets_products", "datasheet:", (item) => {
name: item.data.name,
description: item.data.description,
source: defaultValue(item.data.source, "unknown"), // FIXME: Temporary workaround for old data
url: item.data.url
url: item.data.url,
last_updated: item.updatedAt
};
} else {
console.warn(`[warn] Item does not have a URL: ${item.id}`);
return null;
}
}, {
getLastTimestamp: function () {
return Promise.try(() => {
return knex("datasheets_products")
.orderBy("last_updated", "DESC")
.limit(1);
}).then((results) => {
console.log({results});
if (results.length > 0) {
return results[0].last_updated;
}
});
}
});
const getDatasheetCount = moize(() => {
@ -56,7 +70,7 @@ app.get("/datasheets", (req, res) => {
});
});
app.post("/search", (req, res) => {
app.post("/datasheets/search", (req, res) => {
return Promise.try(() => {
// return knex.raw(`
// SELECT
@ -86,4 +100,8 @@ app.post("/search", (req, res) => {
});
});
app.get("/contact", (req, res) => {
res.render("contact");
});
module.exports = app;

20
src/css/style.css

@ -61,6 +61,17 @@ html, body {
color: teal;
font-size: 1.3em;
}
.betaTag {
width: 1px; /* Out-of-box alignment hack */
position: absolute;
right: -.3em;
bottom: 0;
font-style: italic;
color: rgb(218, 13, 13);
font-size: 1.3em;
}
}
.counter {
@ -69,3 +80,12 @@ html, body {
font-size: .9em;
text-align: right;
}
.staticContent {
margin: 0 2em;
max-width: 900px;
}
.linkSpacer {
margin: 0 .5em;
}

2
src/frontend/components/datasheet-search.jsx

@ -72,7 +72,7 @@ module.exports = function DatasheetSearch({}) {
cancellationToken.current = axios.CancelToken.source();
if (query.length > 0) {
axios.post("/search", {}, {
axios.post("/datasheets/search", {}, {
params: { query: query }
})
.then((response) => {

64
src/sync/index.js

@ -7,35 +7,45 @@ const simpleSink = require("@promistream/simple-sink");
const updateStream = require("./update-stream");
module.exports = function ({ knex }) {
return function createSynchronizer(tableName, prefix, mapper) {
return pipe([
updateStream({ prefix }),
simpleSink((item) => {
return Promise.try(() => {
console.log("[sync] processing item", item);
return matchValue(item.type, {
item: () => {
let result = mapper(item);
if (result != null) {
return function createSynchronizer(tableName, prefix, mapper, { getLastTimestamp } = {}) {
return Promise.try(() => {
if (getLastTimestamp != null) {
// NOTE: Must be a Date object! FIXME: Change this?
return getLastTimestamp();
} else {
return null;
}
}).then((lastTimestamp) => {
return pipe([
updateStream({ prefix, since: lastTimestamp }),
simpleSink((item) => {
return Promise.try(() => {
// TODO: debug-log item processing?
// console.log("[sync] processing item", item);
return matchValue(item.type, {
item: () => {
let result = mapper(item);
if (result != null) {
return knex(tableName)
.insert(result)
.onConflict("id").merge();
}
},
alias: () => {
return knex(tableName)
.insert(result)
.onConflict("id").merge();
.delete()
.where({ id: item.alias });
},
taskResult: () => {
// Ignore these for now
}
},
alias: () => {
return knex(tableName)
.delete()
.where({ id: item.alias });
},
taskResult: () => {
// Ignore these for now
}
});
}).then(() => {
// FIXME: This placeholder `.then` is necessary to make this work *at all*. Investigate why this isn't working otherwise, and whether that's a bug in simple-sink
});
}).then(() => {
// FIXME: This placeholder `.then` is necessary to make this work *at all*. Investigate why this isn't working otherwise, and whether that's a bug in simple-sink
});
})
]).read();
})
]).read();
});
};
};

8
src/sync/update-stream.js

@ -2,6 +2,7 @@
const Promise = require("bluebird");
const bhttp = require("bhttp");
const asBuffer = require("as-buffer");
const pipe = require("@promistream/pipe");
const simpleSource = require("@promistream/simple-source");
@ -11,9 +12,8 @@ const fromNodeStream = require("@promistream/from-node-stream");
const createNDJSONParseStream = require("./ndjson-parse-stream");
module.exports = function createUpdateStream({ prefix } = {}) {
let lastTimestamp = new Date(0);
// let lastTimestamp = new Date();
module.exports = function createUpdateStream({ since, prefix } = {}) {
let lastTimestamp = since ?? new Date(0);
return pipe([
simpleSource(() => {
@ -22,8 +22,6 @@ module.exports = function createUpdateStream({ prefix } = {}) {
// To ensure that we don't hammer the srap instance
return Promise.delay(5 * 1000);
}).then(() => {
// console.log({ lastTimestamp });
// console.log(`http://localhost:3000/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`);
return bhttp.get(`http://localhost:3000/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`, {
stream: true
});

43
src/views/_layout.jsx

@ -0,0 +1,43 @@
"use strict";
const React = require("react");
module.exports = function Layout({ children }) {
return (
<html lang="en">
<head>
<meta charSet="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<title>seekseek</title>
<link rel="stylesheet" href="/css/style.css"/>
</head>
<body>
<div className="header">
<div className="wrapper">
<a className="logoContainer" href="/">
<img src="/images/logo.svg" alt="seekseek logo" className="logo"/>
<span className="siteTag">datasheets</span>
<span className="betaTag">beta</span>
</a>
</div>
</div>
<div className="contents">
{children}
</div>
<div className="footer">
<div className="wrapper">
<a href="https://matrix.to/#/#seekseek:pixie.town?via=pixie.town&via=matrix.org&via=librepush.net" className="chat">
Come chat with us!
</a>
<span className="linkSpacer"></span>
<a href="/contact" className="chat">
Contact/Abuse
</a>
</div>
</div>
<script src="/js/bundle.js"></script>
</body>
</html>
);
};

33
src/views/contact.jsx

@ -0,0 +1,33 @@
"use strict";
const React = require("react");
const Layout = require("./_layout");
module.exports = function Contact() {
return (
<Layout>
<div className="staticContent">
<h2>Do you have questions about SeekSeek, or do you want to contribute to the project?</h2>
<p>Please <a href="https://matrix.to/#/#seekseek:pixie.town?via=pixie.town&via=matrix.org&via=librepush.net">join us in our Matrix room</a>!</p>
<p>We actively watch the channel, and we're always happy to answer any questions you might have. If you have a criticism, we encourage you to share that too! The only requirement is that you do so constructively and respectfully. SeekSeek is a community project, and your feedback is crucial to its success!</p>
<p>If you'd like to work on SeekSeek with us, it's also a good place to start - let's have a chat and see whether it'd be a good fit. Technical skills, while useful, are not required; we also often need people to do data entry and research work, for example.</p>
<h2>Are you a component distributor or manufacturer?</h2>
<p>Please get in touch! SeekSeek is a non-commercial public service, and we want to make sure that we're not accidentally causing harm. We try to scrape very conservatively and prefer sources that require little server resources on your end, so as to not cause load issues - but it's always possible that we overlooked something.</p>
<p>You can reach one of us directly by e-mail at <a href="mailto:admin+seekseek.cryto.net">admin+seekseek@cryto.net</a>, though you are of course also welcome to join our Matrix room.</p>
<p><strong>If our scrapers are causing technical issues for you:</strong> Please tell us what the problem is on your end, and what you think the best solution would be. Our scraping infrastructure is <em>very</em> flexible, and we can almost certainly accommodate your preference. We can also deal with custom feed protocols, even when they don't use HTTP.</p>
<p><strong>If you have concerns about the accuracy or recency of our results:</strong> Please tell us what the best way would be to correct the issue. We try to avoid such issues by only scraping from reputable component distributors and manufacturers directly (preferring the latter when there are multiple sources), but nevertheless we sometimes have outdated results. We're open to doing as much manual correction work as our volunteer-based model allows.</p>
<p><strong>If you are a component seller, and interested in having your prices listed:</strong> We're working on expanding the search with live component prices and stock levels across distributors. Unfortunately, we cannot keep this data fresh enough without a price feed - extracting this data continuously from product listings would likely create an unacceptable load on your end.</p>
<p>If you would like to be listed, please reach out with information about where we can access your price feeds! As SeekSeek is a public service, being listed is completely free of charge, and we do not offer any special advertising positions - though of course, a donation to keep the lights on would be much appreciated. If a regular donation is organizationally difficult for you and a purchasing invoice is required, you can also choose to hire one of our maintainers to implement the price feed integration, for example.</p>
<p><strong>If you would like to request a copyright takedown:</strong> If you are the copyright holder of datasheet(s) listed in our search, you can of course request that we remove these datasheets from the results, and we will do so if your report is valid.</p>
<p>However, we want to ask that you talk with us about it first - we are very open to addressing any practical concerns you may have, and it's not good for <em>anybody</em> to just remove them entirely. Your customers will find it more difficult to find documentation on your products, and that will do no good for your business either!</p>
</div>
</Layout>
);
};

45
src/views/datasheets/index.jsx

@ -2,44 +2,19 @@
const React = require("react");
const Layout = require("../_layout");
module.exports = function Index({ datasheetCount }) {
return (
<html lang="en">
<head>
<meta charSet="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<title>seekseek</title>
<link rel="stylesheet" href="/css/style.css"/>
</head>
<body>
<div className="header">
<div className="wrapper">
<div className="logoContainer">
<img src="/images/logo.svg" alt="seekseek logo" className="logo"/>
<div className="siteTag">datasheets</div>
</div>
</div>
</div>
<div className="contents">
<div className="wrapper">
<div className="counter">
Searching {datasheetCount} datasheets!
</div>
<div id="datasheetSearch">
Loading, please wait...
</div>
</div>
<Layout>
<div className="wrapper">
<div className="counter">
Searching {datasheetCount} datasheets!
</div>
<div className="footer">
<div className="wrapper">
<a href="https://matrix.to/#/#seekseek:pixie.town?via=pixie.town&via=matrix.org&via=librepush.net" className="chat">
Come chat with us!
</a>
</div>
<div id="datasheetSearch">
Loading, please wait...
</div>
<script src="/js/bundle.js"></script>
</body>
</html>
</div>
</Layout>
);
};

12
yarn.lock

@ -1756,6 +1756,13 @@ arrify@^1.0.1:
resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=
as-buffer@^1.6.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/as-buffer/-/as-buffer-1.6.0.tgz#4dbc36da12d0ce988c4015350b193462438c905c"
integrity sha512-K+OjAdZ4SISBSjPyPwE6XMfCqnbq3N5e3k7we5EQk4L1vZziSWb30x2Jb8cuktKt05cyZBBd7fQhdNnjVGbypg==
dependencies:
es6-promise "^4.1.1"
as-expression@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/as-expression/-/as-expression-1.0.0.tgz#7bc620ca4cb2fe0ee90d86729bd6add33b8fd831"
@ -3226,6 +3233,11 @@ es6-promise-try@0.0.1:
resolved "https://registry.yarnpkg.com/es6-promise-try/-/es6-promise-try-0.0.1.tgz#10f140dad27459cef949973e5d21a087f7274b20"
integrity sha1-EPFA2tJ0Wc75SZc+XSGgh/cnSyA=
es6-promise@^4.1.1:
version "4.2.8"
resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a"
integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==
es6-promisify@^6.0.0:
version "6.1.1"
resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-6.1.1.tgz#46837651b7b06bf6fff893d03f29393668d01621"

Loading…
Cancel
Save