// FIXME: Stream reuse via pooling. Instead of spawning a new stream every value, reuse existing streams from a pool, and hold upstream reads instead of sending EndOfStreams. This is not only useful for better performance, but also ensures that streams don't see EOS (and therefore don't do cleanup) until we're *actually* done with the entire thing - this matters especially when dealing with clones with shared state.
// FIXME: Stream reuse via pooling. Instead of spawning a new stream every value, reuse existing streams from a pool, and hold upstream reads instead of sending EndOfStreams. This is not only useful for better performance, but also ensures that streams don't see EOS (and therefore don't do cleanup) until we're *actually* done with the entire thing - this matters especially when dealing with clones with shared state.
// TODO: Make `clone` an optional part of the stream API and use that internally in this module to instantiate clones as-needed from an individual stream rather than a stream factory. Need to figure out, though, how to do resource management in that case - need to ensure that when the upstream ends, not just the clones but also the original input stream receive an EndOfStream signal.
// TODO: Make `clone` an optional part of the stream API and use that internally in this module to instantiate clones as-needed from an individual stream rather than a stream factory. Need to figure out, though, how to do resource management in that case - need to ensure that when the upstream ends, not just the clones but also the original input stream receive an EndOfStream signal.
// FIXME: Verify that all streams in the pool receive an EndOfStream when upstream ends, and have a chance to finalize their internal processing
throwunreachable("Stream wrapper does not exist in mapping");
}
});
});
});
});
}
}
@ -41,42 +60,53 @@ module.exports = function createDynamicStream(streamPickerFunc) {
functiontryHandleQueue(){
functiontryHandleQueue(){
// FIXME: It's possible that a pipeline results in *no* items and so we come up short on pipelines anyway, need to compensate for this by starting a new pipeline
// FIXME: It's possible that a pipeline results in *no* items and so we come up short on pipelines anyway, need to compensate for this by starting a new pipeline
if(requestQueue.length>0){
if(requestQueue.length>0){
if(isRunning===false){
// FIXME: Parallelism, locking? We can't sequentialize the entire thing because we *do* want to accept parallel reads from downstream (so that necessary tasks can be queued behind the scenes), it's just the fetching values from selected pipelines that we want to sequentalize across the stream queue
// FIXME: Parallelism, locking? We can't sequentialize the entire thing because we *do* want to accept parallel reads from downstream (so that necessary tasks can be queued behind the scenes), it's just the fetching values from selected pipelines that we want to sequentalize across the stream queue
// Can probably do this with a good ol' recursive loop and an isRunning marker -- is there a way to abstract this out? We seem to be using this pattern in a lot of different places
// Can probably do this with a good ol' recursive loop and an isRunning marker -- is there a way to abstract this out? We seem to be using this pattern in a lot of different places
debug(`Attaching read operation to request ID ${valueID(resolveRequest)}`);
resolveRequest(read);
resolveRequest(read);
returnPromise.try(()=>{
returnPromise.try(()=>{
returnread;
returnread;
}).catch(()=>{
}).catch(()=>{
// NOTE: We treat any kind of error as a 'completed read' - the error will come out of the request that the read has been attached to and that is where the consumer can handle it. We only care *here* about whether it is time to start another read attempt.
// NOTE: We treat any kind of error as a 'completed read' - the error will come out of the request that the read has been attached to and that is where the consumer can handle it. We only care *here* about whether it is time to start another read attempt.
debug(`Created new pipeline with ID ${valueID(pipeline)}`);
// This is needed to correlate it back later, when the acquired stream ends and we want to release it back into the pool
wrapperToStream.set(pipeline,acquiredStream);
debug(`[stream ${valueID(pipeline)}] Created`);
pipeline.read();// Set the pre-reader in motion
pipeline.read();// Set the pre-reader in motion
streamQueue.push(pipeline);
streamQueue.push(pipeline);
@ -93,7 +123,7 @@ module.exports = function createDynamicStream(streamPickerFunc) {
let{promise,resolve}=pDefer();
let{promise,resolve}=pDefer();
requestQueue.push(resolve);
requestQueue.push(resolve);
debug(`Queued request ID${valueID(resolve)}`);
debug(`[request${valueID(resolve)}] Created`);
returnPromise.try(()=>{
returnPromise.try(()=>{
if(requestQueue.length>streamQueue.length-1){
if(requestQueue.length>streamQueue.length-1){
@ -101,11 +131,11 @@ module.exports = function createDynamicStream(streamPickerFunc) {
// NOTE: We do "-1" here to compensate for the fact that after the first read, there is likely to be an ended stream in the stream queue that we don't know has ended yet, and so this could would otherwise always believe that no streams need to be created and then immediately run into an EndOfStream when attempting to actually *use* the available stream. FIXME: Do we need to make this margin $concurrentReadCount instead of a fixed value of 1?
// NOTE: We do "-1" here to compensate for the fact that after the first read, there is likely to be an ended stream in the stream queue that we don't know has ended yet, and so this could would otherwise always believe that no streams need to be created and then immediately run into an EndOfStream when attempting to actually *use* the available stream. FIXME: Do we need to make this margin $concurrentReadCount instead of a fixed value of 1?
// TODO: Queue multiple pipelines, in case one request eats through multiple pipelines due to 0-value streams?
// TODO: Queue multiple pipelines, in case one request eats through multiple pipelines due to 0-value streams?
// NOTE: startPipeline will produce an EndOfStream if the source runs out of values!
// NOTE: startPipeline will produce an EndOfStream if the source runs out of values!
