wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add option to define I/O size

This commit is contained in:
Chaoguang Lin 2021-02-02 12:39:05 -08:00
parent c25a3f11c3
commit 17671604c0
6 changed files with 33 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
fdbrpc/AsyncFileCached.actor.h
View File

@ -21,6 +21,7 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
#include "fdbserver/Knobs.h"
#if defined(NO_INTELLISENSE) && !defined(FLOW_ASYNCFILECACHED_ACTOR_G_H)
#define FLOW_ASYNCFILECACHED_ACTOR_G_H
#include "fdbrpc/AsyncFileCached.actor.g.h"
@ -519,8 +520,16 @@ struct AFCPage : public EvictablePage, public FastAllocated<AFCPage> {
if (dirty) {
// Wait for rate control if it is set
if (self->owner->getRateControl())
wait(self->owner->getRateControl()->getAllowance(1));
if (self->owner->getRateControl()) {
int allowance = 1;
// If I/O size is defined, wait for the calculated I/O quota
if (FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_IO_SIZE > 0) {
allowance = (self->pageCache->pageSize + FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_IO_SIZE - 1) /
FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_IO_SIZE; // round up
ASSERT(allowance > 0);
}
wait(self->owner->getRateControl()->getAllowance(allowance));
}
if ( self->pageOffset + self->pageCache->pageSize > self->owner->length ) {
ASSERT(self->pageOffset < self->owner->length);

7
fdbserver/KeyValueStoreSQLite.actor.cpp
View File

@ -1361,12 +1361,13 @@ void SQLiteDB::open(bool writable) {
if (walFile.isError()) throw walFile.getError(); // If we've failed to open the file, throw an exception
// Set Rate control if FLOW_KNOBS are positive
if (FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_WINDOW_LIMIT > 0 && FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS > 0) {
if (SERVER_KNOBS->SQLITE_CACHEDFILE_WRITE_WINDOW_LIMIT > 0 &&
SERVER_KNOBS->SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS > 0) {
// The writer thread is created before the readers, so it should initialize the rate controls.
if(writable) {
// Create a new rate control and assign it to both files.
Reference<SpeedLimit> rc(new SpeedLimit(FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_WINDOW_LIMIT,
FLOW_KNOBS->FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS));
Reference<SpeedLimit> rc(new SpeedLimit(SERVER_KNOBS->SQLITE_CACHEDFILE_WRITE_WINDOW_LIMIT,
SERVER_KNOBS->SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS));
dbFile.get()->setRateControl(rc);
walFile.get()->setRateControl(rc);
} else {

11
fdbserver/Knobs.cpp
View File

@ -19,6 +19,7 @@
*/
#include "fdbserver/Knobs.h"
#include "IRandom.h"
#include "fdbrpc/Locality.h"
#include <cmath>
@ -251,6 +252,16 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula
init( SQLITE_CHUNK_SIZE_PAGES, 25600 ); // 100MB
init( SQLITE_CHUNK_SIZE_PAGES_SIM, 1024 ); // 4MB
init( SQLITE_READER_THREADS, 64 ); // number of read threads
init( SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS, -1 );
init( SQLITE_CACHEDFILE_WRITE_WINDOW_LIMIT, -1 );
if( randomize && BUGGIFY ) {
// Choose an window between .01 and 1.01 seconds.
SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS = 0.01 + deterministicRandom()->random01();
// Choose 10k to 50k operations per second
int opsPerSecond = deterministicRandom()->randomInt(1000, 5000);
// Set window limit to opsPerSecond scaled down to window size
SQLITE_CACHEDFILE_WRITE_WINDOW_LIMIT = opsPerSecond * SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS;
}
// Maximum and minimum cell payload bytes allowed on primary page as calculated in SQLite.
// These formulas are copied from SQLite, using its hardcoded constants, so if you are

2
fdbserver/Knobs.h
View File

@ -223,6 +223,8 @@ public:
int SQLITE_CHUNK_SIZE_PAGES;
int SQLITE_CHUNK_SIZE_PAGES_SIM;
int SQLITE_READER_THREADS;
int SQLITE_CACHEDFILE_WRITE_WINDOW_LIMIT;
double SQLITE_CACHEDFILE_WRITE_WINDOW_SECONDS;
// KeyValueStoreSqlite spring cleaning
double SPRING_CLEANING_NO_ACTION_INTERVAL;

13
flow/Knobs.cpp
View File

@ -92,15 +92,10 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
init( MAX_EVICT_ATTEMPTS, 100 ); if( randomize && BUGGIFY ) MAX_EVICT_ATTEMPTS = 2;
init( CACHE_EVICTION_POLICY, "random" );
init( PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION, 0.1 ); if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 0.0; else if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 1.0;
init( FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS, -1 );
init( FLOW_CACHEDFILE_WRITE_WINDOW_LIMIT, -1 );
if( randomize && BUGGIFY ) {
// Choose an window between .01 and 1.01 seconds.
FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS = 0.01 + deterministicRandom()->random01();
// Choose 10k to 50k operations per second
int opsPerSecond = deterministicRandom()->randomInt(1000, 5000);
// Set window limit to opsPerSecond scaled down to window size
FLOW_CACHEDFILE_WRITE_WINDOW_LIMIT = opsPerSecond * FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS;
init( FLOW_CACHEDFILE_WRITE_IO_SIZE, -1 );
if ( randomize && BUGGIFY) {
// Choose 16KB to 64KB as I/O size
FLOW_CACHEDFILE_WRITE_IO_SIZE = deterministicRandom()->randomInt(16384, 65537);
}
//AsyncFileEIO

3
flow/Knobs.h
View File

@ -112,8 +112,7 @@ public:
double PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION;
double TOO_MANY_CONNECTIONS_CLOSED_RESET_DELAY;
int TOO_MANY_CONNECTIONS_CLOSED_TIMEOUT;
int FLOW_CACHEDFILE_WRITE_WINDOW_LIMIT;
double FLOW_CACHEDFILE_WRITE_WINDOW_SECONDS;
int FLOW_CACHEDFILE_WRITE_IO_SIZE;
//AsyncFileEIO
int EIO_MAX_PARALLELISM;