wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Another refactor of SQLite injected error handling to address failures of the previous attempt. Some code cleanup. Added documentation of injected error handling works and why.

This commit is contained in:
Steve Atherton 2021-01-17 04:17:13 -08:00
parent f1adafaf8c
commit 11927f6fff
6 changed files with 48 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
fdbserver/KeyValueStoreSQLite.actor.cpp
View File

@ -261,30 +261,13 @@ struct SQLiteDB : NonCopyable {
}
}
// Consume and return the first of any injected errors detected by the VFSAsyncFile class.
bool consumeInjectedErrors(bool isOpen) {
// Get pointers to the VFSAsyncFile instances that SQLite is using to access its DB and WAL files
// in order to check if either of them have encountered an injected fault.
//
// These could be made into members, but this code is only run when an error occurs and only in simulation.
VFSAsyncFile *vfsDB = (VFSAsyncFile *)sqlite3_get_vfs_db(db);
VFSAsyncFile *vfsWAL = (VFSAsyncFile *)sqlite3_get_vfs_wal(db);
// Return the first error found, leave the others unconsumed.
// Consume the open error last to prevent an injected open error from hiding a non-injected IO op error.
return (vfsDB != nullptr && vfsDB->consumeInjectedError())
|| (vfsWAL != nullptr && vfsWAL->consumeInjectedError())
|| (isOpen && VFSAsyncFile::consumeInjectedOpenError());
}
void checkError( const char* context, int rc ) {
//if (deterministicRandom()->random01() < .001) rc = SQLITE_INTERRUPT;
if (rc) {
// Our exceptions don't propagate through sqlite, so we don't know for sure if the error that caused this was
// an injected fault. Assume that if fault injection is happening, this is an injected fault.
Error err = io_error();
if (g_network->isSimulated() && consumeInjectedErrors(strcmp(context, "open") == 0)) {
if (g_network->isSimulated() && VFSAsyncFile::checkInjectedError()) {
err = err.asInjectedFault();
}

24
fdbserver/VFSAsync.cpp
View File

@ -61,7 +61,7 @@
const uint32_t RESERVED_COUNT = 1U<<29;
VFSAsyncFile::VFSAsyncFile(std::string const& filename, int flags)
: filename(filename), flags(flags), pLockCount(&filename_lockCount_openCount[filename].first), debug_zcrefs(0), debug_zcreads(0), debug_reads(0), chunkSize(0), injectedError(false) {
: filename(filename), flags(flags), pLockCount(&filename_lockCount_openCount[filename].first), debug_zcrefs(0), debug_zcreads(0), debug_reads(0), chunkSize(0) {
filename_lockCount_openCount[filename].second++;
}
@ -92,7 +92,7 @@ static int asyncRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOf
return SQLITE_OK;
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_READ);
}
return SQLITE_IOERR_READ;
}
@ -106,7 +106,7 @@ static int asyncReleaseZeroCopy(sqlite3_file* pFile, void* data, int iAmt, sqlit
p->file->releaseZeroCopy( data, iAmt, iOfst );
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR);
}
return SQLITE_IOERR;
}
@ -131,7 +131,7 @@ static int asyncReadZeroCopy(sqlite3_file *pFile, void **data, int iAmt, sqlite_
return SQLITE_OK;
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_READ);
}
return SQLITE_IOERR_READ;
}
@ -151,7 +151,7 @@ static int asyncReadZeroCopy(sqlite3_file *pFile, void **data, int iAmt, sqlite_
return SQLITE_OK;
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->errorInjected = true;
VFSAsyncFile::setInjectedError(SQLITE_IOERR_READ);
}
return SQLITE_IOERR_READ;
}
@ -170,7 +170,7 @@ static int asyncWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_in
return SQLITE_OK;
} catch(Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_WRITE);
}
return SQLITE_IOERR_WRITE;
}
@ -189,7 +189,7 @@ static int asyncTruncate(sqlite3_file *pFile, sqlite_int64 size){
return SQLITE_OK;
} catch(Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_TRUNCATE);
}
return SQLITE_IOERR_TRUNCATE;
}
@ -202,7 +202,7 @@ static int asyncSync(sqlite3_file *pFile, int flags){
return SQLITE_OK;
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_FSYNC);
}
TraceEvent("VFSSyncError")
@ -225,7 +225,7 @@ static int VFSAsyncFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
return SQLITE_OK;
} catch (Error &e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pFile)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR_FSTAT);
}
return SQLITE_IOERR_FSTAT;
}
@ -548,7 +548,7 @@ static int asyncOpen(
.detail("IAsyncFile", DEBUG_DETERMINISM ? 0 : (int64_t)p->file.getPtr());*/
} catch (Error& e) {
if(e.isInjectedFault()) {
VFSAsyncFile::setOpenError();
VFSAsyncFile::setInjectedError(SQLITE_CANTOPEN);
}
TraceEvent("SQLiteOpenFail").error(e).detail("Filename", p->filename);
p->~VFSAsyncFile();
@ -655,7 +655,7 @@ static int asyncFullPathname(
return SQLITE_OK;
} catch (Error& e) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pVfs)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_IOERR);
}
TraceEvent(SevError,"VFSAsyncFullPathnameError").error(e).detail("PathIn", (std::string)zPath);
return SQLITE_IOERR;
@ -726,7 +726,7 @@ static int asyncSleep(sqlite3_vfs *pVfs, int microseconds){
return microseconds;
} catch( Error &e ) {
if(e.isInjectedFault()) {
((VFSAsyncFile *)pVfs)->setInjectedError();
VFSAsyncFile::setInjectedError(SQLITE_ERROR);
}
TraceEvent(SevError, "AsyncSleepError").error(e,true);
return 0;

82
fdbserver/VFSAsync.h
View File

@ -22,25 +22,6 @@
#include <string>
#include <map>
#include "fdbrpc/IAsyncFile.h"
#include "fdbserver/CoroFlow.h"
//#include <assert.h>
//#include <string.h>
//#ifdef WIN32
//#include <Windows.h>
//#endif
#ifdef __unixish__
//#include <sys/types.h>
//#include <sys/stat.h>
//#include <sys/file.h>
//#include <sys/param.h>
//#include <sys/time.h>
//#include <unistd.h>
//#include <errno.h>
//#include <fcntl.h>
#endif
/*
** When using this VFS, the sqlite3_file* handles that SQLite uses are
@ -52,39 +33,44 @@ struct VFSAsyncFile {
int flags;
std::string filename;
Reference<IAsyncFile> file;
bool injectedError;
// Method to set the injectedError flag to indicate that an injected error has been caught.
// Mostly useful for debugging injected fault handling.
void setInjectedError() {
//TraceEvent(SevInfo, "VFSSetInjectedFault").backtrace().detail("This", (void *)this);
injectedError = true;
}
bool consumeInjectedError() {
bool e = injectedError;
injectedError = false;
return e;
}
// Static methods for capturing an injected open error and consuming it to convert SQLite
// errors into Error instances with isInjectedFault() set.
// The functions setInjectedError() and checkInjectedError() use an INetwork global to store the last
// return code from VFSAsyncFile method resulting from catching an injected Error exception. This allows
// callers of the SQLite API to determine if an error code returned appears to be due to an injected
// error in simulation.
//
// Uses a g_network global so that injected errors on one simulated process do not mask
// non-injected errors on another.
static void setOpenError() {
g_network->setGlobal(INetwork::enSQLiteOpenError, (flowGlobalType)true);
// This scheme is not perfect, as it is possible for non-injected errors to occur after injected errors
// and be incorrectly recognized as injected. This problem already existed, however, as the previous scheme
// assumed that any SQLite error that occurred after any injected error in the simulated process was
// itself injected.
//
// Unfortunately, there is no easy or reliable way to plumb the injectedness of an error though the return
// path of VFSAsyncFile -> SQLite -> SQLite API calls made by KeyValueStoreSQLite.
//
// An attempt was made to store injected errors in VFSAsyncFile instances and expose a SQLiteDB's file
// instances via the SQLite API. This failed, however, because sometimes files are opened, encounter an
// error, and are closed within the lifetime of one SQLite API call so the caller never has an opportunity
// to access the VFSAsyncFile object or its injected error state.
//
// An attempt was also made to match SQLite API return codes to VFSAsyncFile injected error return codes on
// a 1:1 basis, only flagging a code as rejected if it matches the last injected error code and only once.
// This would have been more accurate (though coincidences could occur). This scheme also failed, however,
// because sometimes errors from SQLite APIs are temporarily ignored, relying on a subsequent API call to error,
// however this error could be different and would not have been produced directly by VFSAsyncFile.
//
static void setInjectedError(int64_t rc) {
g_network->setGlobal(INetwork::enSQLiteInjectedError, (flowGlobalType)rc);
TraceEvent("VFSSetInjectedError").detail("ErrorCode", rc).detail("NetworkPtr", (void *)g_network).backtrace();
}
static void clearOpenError() {
g_network->setGlobal(INetwork::enSQLiteOpenError, (flowGlobalType)false);
}
static bool consumeInjectedOpenError() {
bool e = (bool)g_network->global(INetwork::enSQLiteOpenError);
if(e) {
clearOpenError();
}
static bool checkInjectedError() {
// Error code is only checked for non-zero because the SQLite API error code after an injected error
// may not match the error code returned by VFSAsyncFile when the inject error occurred.
bool e = g_network->global(INetwork::enSQLiteInjectedError) != (flowGlobalType)0;
TraceEvent("VFSCheckInjectedError")
.detail("Found", e)
.detail("ErrorCode", (int64_t)g_network->global(INetwork::enSQLiteInjectedError))
.backtrace();
return e;
}

10
fdbserver/sqlite/sqlite3.amalgamation.c
View File

@ -93791,16 +93791,6 @@ SQLITE_API int sqlite3_open_v2(
return openDatabase(filename, ppDb, flags, zVfs);
}
SQLITE_API void * sqlite3_get_vfs_db(sqlite3 *pDb) {
Pager *pPager = sqlite3BtreePager(pDb->aDb[0].pBt);
return (pPager != NULL) ? pPager->fd : NULL;
}
SQLITE_API void * sqlite3_get_vfs_wal(sqlite3 *pDb) {
Pager *pPager = sqlite3BtreePager(pDb->aDb[0].pBt);
return ((pPager != NULL) && (pPager->pWal != NULL)) ? pPager->pWal->pWalFd : NULL;
}
#ifndef SQLITE_OMIT_UTF16
/*
** Open a new database handle.

3
fdbserver/sqlite/sqlite3.h
View File

@ -2411,9 +2411,6 @@ SQLITE_API int sqlite3_open_v2(
const char *zVfs /* Name of VFS module to use */
);
SQLITE_API void * sqlite3_get_vfs_db(sqlite3 *pDb);
SQLITE_API void * sqlite3_get_vfs_wal(sqlite3 *pDb);
/*
** CAPI3REF: Error Codes And Messages
**

2
flow/network.h
View File

@ -452,7 +452,7 @@ public:
enBlobCredentialFiles = 10,
enNetworkAddressesFunc = 11,
enClientFailureMonitor = 12,
enSQLiteOpenError = 13
enSQLiteInjectedError = 13
};
virtual void longTaskCheck( const char* name ) {}