dnrops
/
cryptpad
mirror of https://github.com/xwiki-labs/cryptpad
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce memory usage for the eviction script

This commit is contained in:
yflory 2023-08-21 12:45:18 +02:00
parent 99b4c9b037
commit 5807b4dddf
2 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/eviction.js
View File

@ -367,7 +367,7 @@ module.exports = function (Env, cb) {
TODO make this configurable ?
*/
var BLOOM_CAPACITY = (1 << 22) - 1; // over two million items
var BLOOM_CAPACITY = (1 << 24) - 1; // over two million items
var BLOOM_ERROR = 1 / 10000; // an error rate of one in ten thousand
// we'll use one filter for the set of active documents

23
lib/storage/file.js
View File

@ -584,8 +584,6 @@ var getStats = function (env, channelName, cb) {
// TODO use ../plan.js for a smaller memory footprint
var listChannels = function (root, handler, cb, fast) {
// do twenty things at a time
var sema = Semaphore.create(20);
var dirList = [];
@ -599,17 +597,18 @@ var listChannels = function (root, handler, cb, fast) {
}
dirList = list;
}));
}).nThen(function (w) {
}).nThen(function (waitFor) {
// search inside the nested directories
// stream it so you don't put unnecessary data in memory
var wait = w();
//var wait = w();
var n = nThen;
dirList.forEach(function (dir) {
sema.take(function (give) {
// TODO modify the asynchronous bits here to keep less in memory at any given time
// list a directory -> process its contents with semaphores until less than N jobs are running
// then list the next directory...
// Handle one directory at a time to save some memory
n = n(function (w) {
// do twenty things at a time
var sema = Semaphore.create(20);
var nestedDirPath = Path.join(root, dir);
Fs.readdir(nestedDirPath, w(give(function (err, list) {
Fs.readdir(nestedDirPath, w(function (err, list) {
if (err) { return void handler(err); } // Is this correct?
list.forEach(function (item) {
@ -660,10 +659,10 @@ var listChannels = function (root, handler, cb, fast) {
}, isLonelyMetadata);
});
});
})));
});
}));
}).nThen;
});
wait();
n(waitFor());
}).nThen(function () {
cb();
});