Migrate to SQLite for persistence, and fix native modules in Electron

master
Sven Slootweg 8 years ago
parent 63e7c883f3
commit a5ffac8d75

@ -25,6 +25,7 @@
"file-url": "^1.1.0",
"in-array": "^0.1.2",
"is-function": "^1.0.1",
"knex": "^0.11.5",
"match-object": "0.0.2",
"menubar": "^4.1.1",
"mkdirp": "^0.5.1",
@ -34,6 +35,8 @@
"riot": "^2.4.0",
"riot-query": "0.0.3",
"sanitize-filename": "^1.6.0",
"snake-case": "^1.1.2",
"sqlite3": "^3.1.4",
"xtend": "^4.0.1"
},
"devDependencies": {
@ -44,9 +47,14 @@
"@joepie91/gulp-preset-riot": "^1.0.1",
"@joepie91/gulp-preset-scss": "^1.0.1",
"babel-preset-es2015": "^6.6.0",
"electron-rebuild": "^1.1.5",
"file-url": "^1.1.0",
"gulp": "^3.9.1",
"gulp-livereload": "^3.8.1",
"gulp-rename": "^1.2.2"
},
"scripts": {
"postinstall": "echo 'Rebuilding packages for Electron...' && electron-rebuild && ./rebuild-sqlite3.sh",
"gulp": "gulp"
}
}

@ -0,0 +1,17 @@
detectedArch=`uname -m`
case "$detectedArch" in
i?86) arch="x32" ;;
x86_64) arch="x64" ;;
esac
case "$OSTYPE" in
darwin*) platform="darwin" ;;
linux*) platform="linux" ;;
esac
echo "Rebuilding SQLite3 for Electron... ($platform, $arch)"
npm rebuild --runtime="node" --target="5.1.0" --arch="$arch" sqlite3
mkdir -p node_modules/sqlite3/lib/binding/electron-v1.0-$platform-$arch
cp "node_modules/sqlite3/lib/binding/node-v47-$platform-$arch/node_sqlite3.node" "node_modules/sqlite3/lib/binding/electron-v1.0-$platform-$arch/node_sqlite3.node"

@ -1,7 +1,14 @@
search-results
.result(each="{result, i in opts.results}", class="{active: i === currentSelection}")
h2 {result.name}
.version v{result.latestVersion || "..."}
h2
| {result.name}
span.version v{result.latestVersion || "..."}
.deprecated(if="{result.deprecated}")
strong Deprecated
span.reason {result.deprecated}
.deprecated(if="{result.reserved}")
strong Reserved
span.reason {result.reserved}
.description {result.description || "(no description)"}
script.
@ -34,7 +41,7 @@ search-results
style(scoped, type="scss").
.result {
background-color: white;
color: #676767;
color: #333333;
padding: 9px;
border-top: 1px solid #2a333c;
@ -49,7 +56,7 @@ search-results
h2 {
color: black;
font-size: 21px;
margin: 0px;
margin: 2px 0px;
}
.description {
@ -57,8 +64,34 @@ search-results
}
.version {
margin-left: 8px;
color: gray;
font-style: italic;
float: right;
font-size: 13px;
}
.deprecated {
display: inline-block;
background-color: #b9b9b9;
border-radius: 4px;
font-size: 12px;
margin: 2px 0px;
overflow: hidden;
margin-left: -2px;
strong, .reason {
display: inline-block;
}
strong {
padding: 4px 6px;
background-color: #c70000;
color: #dedede;
}
.reason {
padding: 4px 8px;
color: black;
}
}
}

@ -12,7 +12,7 @@ search
const appDirectory = require("appdirectory");
let appDirectories = new appDirectory("npmbar");
const jsonDB = rfr("lib/db/json-db");
const jsonDB = rfr("lib/db/json-db-sqlite");
const jsonDBCache = rfr("lib/db/json-db-cache");
this.mixin(require("riot-query").mixin);
@ -30,13 +30,15 @@ search
this.on("mount", () => {
Promise.try(() => {
return Promise.all([
jsonDB(path.join(appDirectories.userData(), "search-cache.json")),
jsonDB(path.join(appDirectories.userData(), "package-cache.json"))
jsonDB(path.join(appDirectories.userData(), "search-cache.db")),
jsonDB(path.join(appDirectories.userData(), "package-cache.db"))
]);
}).spread((searchCacheDB, packageCacheDB) => {
let searchCache = jsonDBCache(searchCacheDB, "searchCache");
let packageCache = jsonDBCache(packageCacheDB, "packageCache");
return Promise.all([
jsonDBCache(searchCacheDB, "searchCache"),
jsonDBCache(packageCacheDB, "packageCache")
]);
}).spread((searchCache, packageCache) => {
const search = rfr("lib/search/constructor-io")("CD06z4gVeqSXRiDL2ZNK", searchCache);
const lookupPackage = rfr("lib/package/fetch-metadata")(packageCache);
@ -67,7 +69,19 @@ search
return lookupPackage(result.name);
}).then((metadata) => {
if (dotty.exists(metadata, "dist-tags.latest")) {
result.latestVersion = metadata["dist-tags"].latest;
let latestVersion = metadata["dist-tags"].latest;
let latestMetadata = metadata.versions[latestVersion]
result.latestVersion = latestVersion;
if (latestMetadata != null) {
result.deprecated = latestMetadata.deprecated;
}
if (latestVersion === "0.0.1-security") {
result.reserved = "This formerly popular package was removed, and its name has been reserved."
}
this.update();
}
})

@ -4,19 +4,21 @@ const rfr = require("rfr");
const errors = rfr("lib/util/errors");
module.exports = function(db, collectionName) {
let cacheCollection = db.collection(collectionName);
return {
CacheError: errors.CacheError,
get: function(packageName) {
try {
return cacheCollection.findOne({$cacheKey: packageName});
} catch (err) { // FIXME
throw new errors.CacheError("Not in cache");
return Promise.try(() => {
return db.collection(collectionName);
}).then((cacheCollection) => {
return {
CacheError: errors.CacheError,
get: function(packageName) {
try {
return cacheCollection.findOne({$cacheKey: packageName});
} catch (err) { // FIXME
throw new errors.CacheError("Not in cache");
}
},
set: function(packageName, metadata) {
return cacheCollection.upsertBy("$cacheKey", metadata);
}
},
set: function(packageName, metadata) {
return cacheCollection.upsertBy("$cacheKey", metadata);
}
}
});
}

@ -0,0 +1,164 @@
'use strict';
const Promise = require("bluebird");
const knex = require("knex");
const fs = Promise.promisifyAll(require("fs"));
const matchObject = require("match-object");
const uuid = require("uuid");
const pick = require("object-pick");
const snakeCase = require("snake-case");
const rfr = require("rfr");
const knexErrors = rfr("lib/db/knex-error-type");
function loadCollection(db, collection) {
return Promise.try(() => {
return db(collection).select();
}).then((results) => {
return results.map((result) => {
return Object.assign(JSON.parse(result.value), {
$id: result.id
});
});
});
}
module.exports = function(path) {
let collections = {};
let db = knex({
client: "sqlite3",
connection: {
filename: path
},
useNullAsDefault: true,
debug: true
});
function createTable(name) {
return db.schema.createTable(name, function(table) {
table.text("id");
table.text("value");
});
}
function getCollection(name) {
let indexes; // TODO
function findIdIndex(id) {
if (id == null) {
return null;
}
return collections[name].findIndex((item) => item.$id === id);
}
return {
find: function(query) {
return collections[name].filter((item) => matchObject(query, item));
},
findOne: function(query) {
let result = collections[name].find((item) => matchObject(query, item));
if (result == null) {
throw new Error("No results found");
} else {
return result;
}
},
insert: function(object, options = {}) {
/* Intentional mutation. */
Object.assign(object, {$id: uuid.v4()});
collections[name].push(object);
return db(name).insert({
id: object.$id,
value: JSON.stringify(object)
}).then(() => {});
},
update: function(object, options = {}) {
let index;
if (index = findIdIndex(object.$id)) {
if (options.patch === true) {
Object.assign(collections[name][index], object);
} else {
collections[name][index] = object;
}
return db(name).where({id: object.$id}).update({
value: JSON.stringify(collections[name][index])
}).then(() => {});
} else {
throw new Error("No such object exists");
}
},
upsert: function(object, options = {}) {
try {
this.update(object, options);
} catch (err) {
this.insert(object, options);
}
},
upsertBy: function(keys, object, options = {}) {
let query = pick(object, keys);
try {
let result = this.findOne(query);
// FIXME: The following shouldn't be in the try block...
object.$id = result.$id;
return this.update(object, options);
} catch (err) {
return this.insert(object, options);
}
},
delete: function(object) {
let index;
if (index = findIdIndex(object.$id)) {
collections[name].splice(index, 1);
return db(name).where({id: object.$id}).delete().then(() => {});
} else {
throw new Error("No such object exists");
}
},
deleteBy: function(query) {
let toRemove = collections[name].filter((item) => matchObject(query, item));
collections[name] = collections[name].filter((item) => (toRemove.indexOf(item) !== -1));
return Promise.map(toRemove, (item) => {
return db(name).where({id: item.$id}).delete();
});
},
ensureIndex: function(property) {
// TODO
}
}
}
let dbAPI = {
close: function() {},
collection: function(name) {
return Promise.try(() => {
let snakeCasedName = snakeCase(name);
if (collections[snakeCasedName] != null) {
return getCollection(snakeCasedName);
} else {
return Promise.try(() => {
return loadCollection(db, snakeCasedName);
}).then((collectionData) => {
collections[snakeCasedName] = collectionData;
}).catch(knexErrors.sqlite.NoSuchTable, (err) => {
collections[snakeCasedName] = [];
return createTable(snakeCasedName);
}).then(() => {
return getCollection(snakeCasedName);
});
}
});
}
}
return dbAPI;
}

@ -0,0 +1,9 @@
'use strict';
module.exports = {
sqlite: {
NoSuchTable: function(err) {
return err.message.includes("SQLITE_ERROR: no such table:");
}
}
}
Loading…
Cancel
Save