Improve empty-queue behaviour, reduce stampeding

master
Sven Slootweg 2 years ago
parent 31742f8638
commit 919985bbd2

@ -23,7 +23,8 @@ module.exports = function(state) {
return {
defaultSettings: {
taskBatchSize: 1000,
taskBatchDelay: 30 * 1000
taskBatchDelay: 5 * 60 * 1000, // FIXME: Rename to reflect changed behaviour?
delayRandomization: 0.2
},
create: function createPostgreSQLBackend(options) {
let knex = knexLibrary({

@ -143,7 +143,8 @@ module.exports = function ({ metrics, backendSettings, knex }) {
}).then((newItems) => {
if (newItems === 0) {
// TODO: Consider using LISTEN/NOTIFY instead? Worth the added complexity?
return Promise.resolve([]).delay(backendSettings.taskBatchDelay);
let randomization = Math.random() * backendSettings.delayRandomization * backendSettings.taskBatchDelay; // To prevent stampeding by low-throughput tasks
return Promise.resolve([]).delay(backendSettings.taskBatchDelay + randomization);
} else {
// Have another go right away
return [];

Loading…
Cancel
Save