dnrops
/
cryptpad
mirror of https://github.com/xwiki-labs/cryptpad
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update eviction script

This commit is contained in:
yflory 2023-08-23 10:57:11 +02:00
parent fe7d48b0f6
commit ce572e813e
2 changed files with 48 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
lib/eviction.js
View File

@ -186,8 +186,7 @@ var evictArchived = function (Env, cb) {
var handler = function (err, item, cb) {
if (err) {
Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err);
return void cb();
return Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err, cb);
}
// don't mess with files that are freshly stored in cold storage
// based on ctime because that's changed when the file is moved...
@ -199,13 +198,11 @@ var evictArchived = function (Env, cb) {
// expire it
store.removeArchivedChannel(item.channel, w(function (err) {
if (err) {
Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
return Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
error: err,
channel: item.channel,
});
return void cb();
}, cb);
}
Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel);
if (item.channel.length === 32) {
removed++;
@ -213,7 +210,7 @@ var evictArchived = function (Env, cb) {
accounts++;
}
cb();
Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel, cb);
}));
};
@ -399,8 +396,7 @@ module.exports = function (Env, cb) {
}
if (err) {
Log.error('EVICT_CHANNEL_CATEGORIZATION', err);
return void cb();
return Log.error('EVICT_CHANNEL_CATEGORIZATION', err, cb);
}
// if the channel has been modified recently
@ -421,7 +417,7 @@ module.exports = function (Env, cb) {
Log.info('EVICT_CHANNELS_CATEGORIZED', {
active: active,
channels: channels,
});
}, w());
};
Log.info('EVICT_CHANNEL_ACTIVITY_START', 'Assessing channel activity');
@ -443,12 +439,10 @@ module.exports = function (Env, cb) {
}
if (err) {
Log.error("EVICT_BLOB_CATEGORIZATION", err);
return void next();
return Log.error("EVICT_BLOB_CATEGORIZATION", err, next);
}
if (!item) {
next();
return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item);
return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item, next);
}
if (item.mtime > inactiveTime) {
activeDocs.add(item.blobId);
@ -462,7 +456,7 @@ module.exports = function (Env, cb) {
Log.info('EVICT_BLOBS_CATEGORIZED', {
active: active,
blobs: n_blobs,
});
}, w());
}));
};
@ -530,31 +524,27 @@ module.exports = function (Env, cb) {
// we plan to delete them, because it may be interesting information
inactive++;
if (PRESERVE_INACTIVE_ACCOUNTS) {
Log.info('EVICT_INACTIVE_ACCOUNT_PRESERVED', {
pinAll(pinList);
return Log.info('EVICT_INACTIVE_ACCOUNT_PRESERVED', {
id: id,
mtime: mtime,
});
pinAll(pinList);
return void next();
}, next);
}
if (isPremiumAccount(id)) {
Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", {
pinAll(pinList);
return Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", {
id: id,
mtime: mtime,
});
pinAll(pinList);
return void next();
}, next);
}
// remove the pin logs of inactive accounts if inactive account removal is configured
pinStore.archiveChannel(id, function (err) {
if (err) {
Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err);
return void next();
return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next);
}
Log.info('EVICT_INACTIVE_ACCOUNT_LOG', id);
next();
Log.info('EVICT_INACTIVE_ACCOUNT_LOG', id, next);
});
};
@ -588,12 +578,10 @@ module.exports = function (Env, cb) {
blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err);
return void next();
return Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err, next);
}
if (!item) {
next();
return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item);
return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item, next);
}
total++;
if (total % PROGRESS_FACTOR === 0) {
@ -613,21 +601,19 @@ module.exports = function (Env, cb) {
removed++;
blobs.archive.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
return Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
error: err,
item: item,
});
return void next();
}, next);
}
Log.info("EVICT_ARCHIVE_BLOB", {
item: item,
});
next();
}, next);
});
}, w(function () {
report.totalBlobs = total;
report.activeBlobs = total - removed;
Log.info('EVICT_BLOBS_REMOVED', removed);
Log.info('EVICT_BLOBS_REMOVED', removed, w());
}));
};
@ -641,12 +627,10 @@ module.exports = function (Env, cb) {
blobs.list.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next);
}
if (!item) {
next();
return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item);
return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item, next);
}
total++;
@ -662,8 +646,7 @@ module.exports = function (Env, cb) {
blobs.size(item.blobId, w(function (err, size) {
if (err) {
w.abort();
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next);
}
if (size !== 0) {
w.abort();
@ -672,19 +655,18 @@ module.exports = function (Env, cb) {
}));
}).nThen(function () {
blobs.remove.proof(item.safeKey, item.blobId, function (err) {
next();
if (err) {
return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item);
return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next);
}
removed++;
return Log.info("EVICT_BLOB_PROOF_LONELY", item);
return Log.info("EVICT_BLOB_PROOF_LONELY", item, next);
});
});
}, w(function () {
Log.info("EVICT_BLOB_PROOFS_REMOVED", {
removed,
total,
});
}, w());
}));
};
@ -703,8 +685,7 @@ module.exports = function (Env, cb) {
}
if (err) {
Log.error('EVICT_CHANNEL_ITERATION', err);
return void cb();
return Log.error('EVICT_CHANNEL_ITERATION', err, cb);
}
// ignore the special admin broadcast channel
@ -715,14 +696,12 @@ module.exports = function (Env, cb) {
if (item.channel.length === 34) {
return void store.removeChannel(item.channel, w(function (err) {
if (err) {
Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
return Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
error: err,
channel: item.channel,
});
return void cb();
}, cb);
}
Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel);
cb();
Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel, cb);
}));
}
@ -744,20 +723,19 @@ module.exports = function (Env, cb) {
}
// else fall through to the archival
}));
}).nThen(function () {
}).nThen(function (w) {
return void store.archiveChannel(item.channel, w(function (err) {
if (err) {
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
error: err,
channel: item.channel,
});
return void cb();
}, w());
return;
}
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel);
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w());
archived++;
cb();
}));
});
}).nThen(cb);
};
var done = function () {

15
lib/log.js
View File

@ -1,5 +1,6 @@
/*jshint esversion: 6 */
var Store = require("./storage/file");
var Util = require("../common-util");
var Logger = module.exports;
@ -15,9 +16,13 @@ var messageTemplate = function (type, time, tag, info) {
var noop = function () {};
var write = function (ctx, content) {
if (!ctx.store) { return; }
ctx.store.log(ctx.channelName, content, noop);
var write = function (ctx, content, cb) {
if (typeof(cb) !== "function") { cb = noop; }
if (!ctx.store) {
cb = Util.mkAsync(cb);
return void cb();
}
ctx.store.log(ctx.channelName, content, cb);
};
// various degrees of logging
@ -37,7 +42,7 @@ var createLogType = function (ctx, type) {
if (logLevels.indexOf(type) < logLevels.indexOf(ctx.logLevel)) {
return noop;
}
return function (tag, info) {
return function (tag, info, cb) {
if (ctx.shutdown) {
throw new Error("Logger has been shut down!");
}
@ -51,7 +56,7 @@ var createLogType = function (ctx, type) {
if (ctx.logToStdout && typeof(handlers[type]) === 'function') {
handlers[type](ctx, content);
}
write(ctx, content);
write(ctx, content, cb);
};
};