From 78a0c5a051841b4cb7d3d6fed7e02cee8aa4d032 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Fri, 4 Mar 2022 01:42:59 +0100 Subject: [PATCH] WIP --- migrations/20211215172404_store-date.js | 13 ++++ public/css/style.css | 20 ++++++ src/app.js | 22 ++++++- src/css/style.css | 20 ++++++ src/frontend/components/datasheet-search.jsx | 2 +- src/sync/index.js | 64 +++++++++++--------- src/sync/update-stream.js | 8 +-- src/views/_layout.jsx | 43 +++++++++++++ src/views/contact.jsx | 33 ++++++++++ src/views/datasheets/index.jsx | 45 +++----------- yarn.lock | 12 ++++ 11 files changed, 212 insertions(+), 70 deletions(-) create mode 100644 migrations/20211215172404_store-date.js create mode 100644 src/views/_layout.jsx create mode 100644 src/views/contact.jsx diff --git a/migrations/20211215172404_store-date.js b/migrations/20211215172404_store-date.js new file mode 100644 index 0000000..fcbe72a --- /dev/null +++ b/migrations/20211215172404_store-date.js @@ -0,0 +1,13 @@ +"use strict"; + +module.exports.up = function(knex, Promise) { + return knex.schema.table("datasheets_products", (table) => { + table.timestamp("last_updated").index().notNull(); + }); +}; + +module.exports.down = function(knex, Promise) { + return knex.schema.table("datasheets_products", (table) => { + table.dropColumn("last_updated"); + }); +}; diff --git a/public/css/style.css b/public/css/style.css index ef2bb3f..da13113 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -60,9 +60,29 @@ html, body { font-size: 1.3em; } +.logoContainer .betaTag { + width: 1px; /* Out-of-box alignment hack */ + position: absolute; + right: -.3em; + bottom: 0; + font-style: italic; + + color: rgb(218, 13, 13); + font-size: 1.3em; + } + .counter { margin-bottom: .5em; font-style: italic; font-size: .9em; text-align: right; } + +.staticContent { + margin: 0 2em; + max-width: 900px; +} + +.linkSpacer { + margin: 0 .5em; +} diff --git a/src/app.js b/src/app.js index 47f8d7d..7aae7f8 100644 --- a/src/app.js +++ b/src/app.js @@ -18,12 +18,26 @@ createSynchronizer("datasheets_products", "datasheet:", (item) => { name: item.data.name, description: item.data.description, source: defaultValue(item.data.source, "unknown"), // FIXME: Temporary workaround for old data - url: item.data.url + url: item.data.url, + last_updated: item.updatedAt }; } else { console.warn(`[warn] Item does not have a URL: ${item.id}`); return null; } +}, { + getLastTimestamp: function () { + return Promise.try(() => { + return knex("datasheets_products") + .orderBy("last_updated", "DESC") + .limit(1); + }).then((results) => { + console.log({results}); + if (results.length > 0) { + return results[0].last_updated; + } + }); + } }); const getDatasheetCount = moize(() => { @@ -56,7 +70,7 @@ app.get("/datasheets", (req, res) => { }); }); -app.post("/search", (req, res) => { +app.post("/datasheets/search", (req, res) => { return Promise.try(() => { // return knex.raw(` // SELECT @@ -86,4 +100,8 @@ app.post("/search", (req, res) => { }); }); +app.get("/contact", (req, res) => { + res.render("contact"); +}); + module.exports = app; diff --git a/src/css/style.css b/src/css/style.css index d799bb8..6ef97ab 100644 --- a/src/css/style.css +++ b/src/css/style.css @@ -61,6 +61,17 @@ html, body { color: teal; font-size: 1.3em; } + + .betaTag { + width: 1px; /* Out-of-box alignment hack */ + position: absolute; + right: -.3em; + bottom: 0; + font-style: italic; + + color: rgb(218, 13, 13); + font-size: 1.3em; + } } .counter { @@ -69,3 +80,12 @@ html, body { font-size: .9em; text-align: right; } + +.staticContent { + margin: 0 2em; + max-width: 900px; +} + +.linkSpacer { + margin: 0 .5em; +} diff --git a/src/frontend/components/datasheet-search.jsx b/src/frontend/components/datasheet-search.jsx index faa1052..4d619d5 100644 --- a/src/frontend/components/datasheet-search.jsx +++ b/src/frontend/components/datasheet-search.jsx @@ -72,7 +72,7 @@ module.exports = function DatasheetSearch({}) { cancellationToken.current = axios.CancelToken.source(); if (query.length > 0) { - axios.post("/search", {}, { + axios.post("/datasheets/search", {}, { params: { query: query } }) .then((response) => { diff --git a/src/sync/index.js b/src/sync/index.js index 0051c84..494e140 100644 --- a/src/sync/index.js +++ b/src/sync/index.js @@ -7,35 +7,45 @@ const simpleSink = require("@promistream/simple-sink"); const updateStream = require("./update-stream"); module.exports = function ({ knex }) { - return function createSynchronizer(tableName, prefix, mapper) { - return pipe([ - updateStream({ prefix }), - simpleSink((item) => { - return Promise.try(() => { - console.log("[sync] processing item", item); - return matchValue(item.type, { - item: () => { - let result = mapper(item); - - if (result != null) { + return function createSynchronizer(tableName, prefix, mapper, { getLastTimestamp } = {}) { + return Promise.try(() => { + if (getLastTimestamp != null) { + // NOTE: Must be a Date object! FIXME: Change this? + return getLastTimestamp(); + } else { + return null; + } + }).then((lastTimestamp) => { + return pipe([ + updateStream({ prefix, since: lastTimestamp }), + simpleSink((item) => { + return Promise.try(() => { + // TODO: debug-log item processing? + // console.log("[sync] processing item", item); + return matchValue(item.type, { + item: () => { + let result = mapper(item); + + if (result != null) { + return knex(tableName) + .insert(result) + .onConflict("id").merge(); + } + }, + alias: () => { return knex(tableName) - .insert(result) - .onConflict("id").merge(); + .delete() + .where({ id: item.alias }); + }, + taskResult: () => { + // Ignore these for now } - }, - alias: () => { - return knex(tableName) - .delete() - .where({ id: item.alias }); - }, - taskResult: () => { - // Ignore these for now - } + }); + }).then(() => { + // FIXME: This placeholder `.then` is necessary to make this work *at all*. Investigate why this isn't working otherwise, and whether that's a bug in simple-sink }); - }).then(() => { - // FIXME: This placeholder `.then` is necessary to make this work *at all*. Investigate why this isn't working otherwise, and whether that's a bug in simple-sink - }); - }) - ]).read(); + }) + ]).read(); + }); }; }; diff --git a/src/sync/update-stream.js b/src/sync/update-stream.js index 5c5b507..2a52d15 100644 --- a/src/sync/update-stream.js +++ b/src/sync/update-stream.js @@ -2,6 +2,7 @@ const Promise = require("bluebird"); const bhttp = require("bhttp"); +const asBuffer = require("as-buffer"); const pipe = require("@promistream/pipe"); const simpleSource = require("@promistream/simple-source"); @@ -11,9 +12,8 @@ const fromNodeStream = require("@promistream/from-node-stream"); const createNDJSONParseStream = require("./ndjson-parse-stream"); -module.exports = function createUpdateStream({ prefix } = {}) { - let lastTimestamp = new Date(0); - // let lastTimestamp = new Date(); +module.exports = function createUpdateStream({ since, prefix } = {}) { + let lastTimestamp = since ?? new Date(0); return pipe([ simpleSource(() => { @@ -22,8 +22,6 @@ module.exports = function createUpdateStream({ prefix } = {}) { // To ensure that we don't hammer the srap instance return Promise.delay(5 * 1000); }).then(() => { - // console.log({ lastTimestamp }); - // console.log(`http://localhost:3000/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`); return bhttp.get(`http://localhost:3000/updates?prefix=${encodeURIComponent(prefix)}&since=${Math.floor(lastTimestamp.getTime())}`, { stream: true }); diff --git a/src/views/_layout.jsx b/src/views/_layout.jsx new file mode 100644 index 0000000..7070189 --- /dev/null +++ b/src/views/_layout.jsx @@ -0,0 +1,43 @@ +"use strict"; + +const React = require("react"); + +module.exports = function Layout({ children }) { + return ( + + + + + + seekseek + + + +
+
+ + seekseek logo + datasheets + beta + +
+
+
+ {children} +
+
+
+ + Come chat with us! + + • + + Contact/Abuse + +
+
+ + + + ); +}; diff --git a/src/views/contact.jsx b/src/views/contact.jsx new file mode 100644 index 0000000..89d04db --- /dev/null +++ b/src/views/contact.jsx @@ -0,0 +1,33 @@ +"use strict"; + +const React = require("react"); + +const Layout = require("./_layout"); + +module.exports = function Contact() { + return ( + +
+

Do you have questions about SeekSeek, or do you want to contribute to the project?

+

Please join us in our Matrix room!

+

We actively watch the channel, and we're always happy to answer any questions you might have. If you have a criticism, we encourage you to share that too! The only requirement is that you do so constructively and respectfully. SeekSeek is a community project, and your feedback is crucial to its success!

+

If you'd like to work on SeekSeek with us, it's also a good place to start - let's have a chat and see whether it'd be a good fit. Technical skills, while useful, are not required; we also often need people to do data entry and research work, for example.

+ +

Are you a component distributor or manufacturer?

+

Please get in touch! SeekSeek is a non-commercial public service, and we want to make sure that we're not accidentally causing harm. We try to scrape very conservatively and prefer sources that require little server resources on your end, so as to not cause load issues - but it's always possible that we overlooked something.

+

You can reach one of us directly by e-mail at admin+seekseek@cryto.net, though you are of course also welcome to join our Matrix room.

+ +

If our scrapers are causing technical issues for you: Please tell us what the problem is on your end, and what you think the best solution would be. Our scraping infrastructure is very flexible, and we can almost certainly accommodate your preference. We can also deal with custom feed protocols, even when they don't use HTTP.

+ +

If you have concerns about the accuracy or recency of our results: Please tell us what the best way would be to correct the issue. We try to avoid such issues by only scraping from reputable component distributors and manufacturers directly (preferring the latter when there are multiple sources), but nevertheless we sometimes have outdated results. We're open to doing as much manual correction work as our volunteer-based model allows.

+ +

If you are a component seller, and interested in having your prices listed: We're working on expanding the search with live component prices and stock levels across distributors. Unfortunately, we cannot keep this data fresh enough without a price feed - extracting this data continuously from product listings would likely create an unacceptable load on your end.

+

If you would like to be listed, please reach out with information about where we can access your price feeds! As SeekSeek is a public service, being listed is completely free of charge, and we do not offer any special advertising positions - though of course, a donation to keep the lights on would be much appreciated. If a regular donation is organizationally difficult for you and a purchasing invoice is required, you can also choose to hire one of our maintainers to implement the price feed integration, for example.

+ +

If you would like to request a copyright takedown: If you are the copyright holder of datasheet(s) listed in our search, you can of course request that we remove these datasheets from the results, and we will do so if your report is valid.

+ +

However, we want to ask that you talk with us about it first - we are very open to addressing any practical concerns you may have, and it's not good for anybody to just remove them entirely. Your customers will find it more difficult to find documentation on your products, and that will do no good for your business either!

+
+ + ); +}; diff --git a/src/views/datasheets/index.jsx b/src/views/datasheets/index.jsx index 43e426f..691a688 100644 --- a/src/views/datasheets/index.jsx +++ b/src/views/datasheets/index.jsx @@ -2,44 +2,19 @@ const React = require("react"); +const Layout = require("../_layout"); + module.exports = function Index({ datasheetCount }) { return ( - - - - - - seekseek - - - -
-
-
- seekseek logo -
datasheets
-
-
-
-
-
-
- Searching {datasheetCount} datasheets! -
-
- Loading, please wait... -
-
+ +
+
+ Searching {datasheetCount} datasheets!
-
- +
+ Loading, please wait...
- - - +
+ ); }; diff --git a/yarn.lock b/yarn.lock index ef832de..38fb426 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1756,6 +1756,13 @@ arrify@^1.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= +as-buffer@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/as-buffer/-/as-buffer-1.6.0.tgz#4dbc36da12d0ce988c4015350b193462438c905c" + integrity sha512-K+OjAdZ4SISBSjPyPwE6XMfCqnbq3N5e3k7we5EQk4L1vZziSWb30x2Jb8cuktKt05cyZBBd7fQhdNnjVGbypg== + dependencies: + es6-promise "^4.1.1" + as-expression@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/as-expression/-/as-expression-1.0.0.tgz#7bc620ca4cb2fe0ee90d86729bd6add33b8fd831" @@ -3226,6 +3233,11 @@ es6-promise-try@0.0.1: resolved "https://registry.yarnpkg.com/es6-promise-try/-/es6-promise-try-0.0.1.tgz#10f140dad27459cef949973e5d21a087f7274b20" integrity sha1-EPFA2tJ0Wc75SZc+XSGgh/cnSyA= +es6-promise@^4.1.1: + version "4.2.8" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a" + integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w== + es6-promisify@^6.0.0: version "6.1.1" resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-6.1.1.tgz#46837651b7b06bf6fff893d03f29393668d01621"