dnrops
/
cryptpad
mirror of https://github.com/xwiki-labs/cryptpad
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

introduce some artificial delays in the eviction script 

...and reduce the margin of error in its bloom filters
This commit is contained in:
ansuz 2020-10-14 17:22:31 +05:30
parent 90f046f896
commit a29c5641b8
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/eviction.js
View File

@ -81,7 +81,9 @@ module.exports = function (Env, cb) {
TODO make this configurable ?
*/
var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items
var BLOOM_ERROR = 1 / 1000; // an error rate of one in a thousand
var BLOOM_ERROR = 1 / 10000; // an error rate of one in a thousand
// the number of ms artificially introduced between CPU-intensive operations
var THROTTLE_FACTOR = 10;
// we'll use one filter for the set of active documents
var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
@ -165,6 +167,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next();
@ -190,6 +193,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
@ -246,6 +250,7 @@ module.exports = function (Env, cb) {
var active = 0;
blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
n_blobs++;
if (err) {
Log.error("EVICT_BLOB_CATEGORIZATION", err);
@ -311,6 +316,7 @@ module.exports = function (Env, cb) {
// otherwise, we'll only retain data from active accounts
// so we need more heuristics
var handler = function (content, id, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
accounts++;
var mtime = content.latest;
@ -416,6 +422,7 @@ module.exports = function (Env, cb) {
// if they don't correspond to a pinned or active file
var removed = 0;
blobs.list.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
@ -458,6 +465,7 @@ module.exports = function (Env, cb) {
var archived = 0;
var handler = function (err, item, cb) {
cb = Util.mkAsync(cb, THROTTLE_FACTOR);
channels++;
if (err) {
Log.error('EVICT_CHANNEL_ITERATION', err);