wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug fixes. Btree now uses page 0 as root and will write initial page only if necessary. Added debug printf macro.

This commit is contained in:
Stephen Atherton 2017-07-14 11:36:49 -07:00
parent d8b82ecbe4
commit fc0557252b
3 changed files with 51 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fdbserver/IPager.h
View File

@ -27,6 +27,8 @@
#include "flow/flow.h" #include "flow/flow.h"
#include "fdbclient/FDBTypes.h" #include "fdbclient/FDBTypes.h"
#define REDWOOD_DEBUG
typedef uint32_t LogicalPageID; // uint64_t? typedef uint32_t LogicalPageID; // uint64_t?
class IPage { class IPage {

48
fdbserver/IndirectShadowPager.actor.cpp
View File

@ -23,6 +23,12 @@
#include "flow/UnitTest.h" #include "flow/UnitTest.h"
#ifdef REDWOOD_DEBUG
#define debug_printf printf
#else
#define debug_printf(...)
#endif
IndirectShadowPage::IndirectShadowPage() : allocated(true) { IndirectShadowPage::IndirectShadowPage() : allocated(true) {
data = (uint8_t*)FastAllocator<4096>::allocate(); data = (uint8_t*)FastAllocator<4096>::allocate();
} }
@ -95,7 +101,7 @@ ACTOR Future<Void> recover(IndirectShadowPager *pager) {
pr >> pagesAllocated; pr >> pagesAllocated;
pager->pagerFile.init(fileSize, pagesAllocated); pager->pagerFile.init(fileSize, pagesAllocated);
fprintf(stderr, "Recovered pages allocated: %d\n", pager->pagerFile.pagesAllocated); debug_printf("Recovered pages allocated: %d\n", pager->pagerFile.pagesAllocated);
ASSERT(pager->pagerFile.pagesAllocated != PagerFile::INVALID_PAGE); ASSERT(pager->pagerFile.pagesAllocated != PagerFile::INVALID_PAGE);
Optional<Value> latestVersionValue = wait(pager->pageTableLog->readValue(IndirectShadowPager::LATEST_VERSION_KEY)); Optional<Value> latestVersionValue = wait(pager->pageTableLog->readValue(IndirectShadowPager::LATEST_VERSION_KEY));
@ -111,7 +117,7 @@ ACTOR Future<Void> recover(IndirectShadowPager *pager) {
vr >> pager->oldestVersion; vr >> pager->oldestVersion;
} }
fprintf(stderr, "Recovered version info: %d - %d\n", pager->oldestVersion, pager->latestVersion); debug_printf("Recovered version info: %d - %d\n", pager->oldestVersion, pager->latestVersion);
pager->committedVersion = pager->latestVersion; pager->committedVersion = pager->latestVersion;
Standalone<VectorRef<KeyValueRef>> tableEntries = wait(pager->pageTableLog->readRange(KeyRangeRef(IndirectShadowPager::TABLE_ENTRY_PREFIX, strinc(IndirectShadowPager::TABLE_ENTRY_PREFIX)))); Standalone<VectorRef<KeyValueRef>> tableEntries = wait(pager->pageTableLog->readRange(KeyRangeRef(IndirectShadowPager::TABLE_ENTRY_PREFIX, strinc(IndirectShadowPager::TABLE_ENTRY_PREFIX))));
@ -130,10 +136,10 @@ ACTOR Future<Void> recover(IndirectShadowPager *pager) {
LogicalPageID pageTableSize = std::max<LogicalPageID>(logicalPageID+1, SERVER_KNOBS->PAGER_RESERVED_PAGES); LogicalPageID pageTableSize = std::max<LogicalPageID>(logicalPageID+1, SERVER_KNOBS->PAGER_RESERVED_PAGES);
pager->pageTable.resize(pageTableSize); pager->pageTable.resize(pageTableSize);
fprintf(stderr, "Recovered page table size: %d\n", pageTableSize); debug_printf("Recovered page table size: %d\n", pageTableSize);
} }
else { else {
fprintf(stderr, "Recovered no page table entries\n"); debug_printf("Recovered no page table entries\n");
} }
LogicalPageID nextPageID = SERVER_KNOBS->PAGER_RESERVED_PAGES; LogicalPageID nextPageID = SERVER_KNOBS->PAGER_RESERVED_PAGES;
@ -172,10 +178,10 @@ ACTOR Future<Void> recover(IndirectShadowPager *pager) {
++nextPageID; ++nextPageID;
} }
fprintf(stderr, "Recovered page table entry %d -> (%d, %d)\n", logicalPageID, version, physicalPageID); debug_printf("Recovered page table entry %d -> (%d, %d)\n", logicalPageID, version, physicalPageID);
} }
fprintf(stderr, "Building physical free list\n"); debug_printf("Building physical free list\n");
// TODO: can we do this better? does it require storing extra info in the log? // TODO: can we do this better? does it require storing extra info in the log?
PhysicalPageID nextPhysicalPageID = 0; PhysicalPageID nextPhysicalPageID = 0;
for(auto itr = allocatedPhysicalPages.begin(); itr != allocatedPhysicalPages.end(); ++itr) { for(auto itr = allocatedPhysicalPages.begin(); itr != allocatedPhysicalPages.end(); ++itr) {
@ -198,7 +204,7 @@ ACTOR Future<Void> recover(IndirectShadowPager *pager) {
pager->pagerFile.finishedMarkingPages(); pager->pagerFile.finishedMarkingPages();
pager->pagerFile.startVacuuming(); pager->pagerFile.startVacuuming();
fprintf(stderr, "Finished recovery\n", pager->oldestVersion); debug_printf("Finished recovery\n", pager->oldestVersion);
TraceEvent("PagerFinishedRecovery"); TraceEvent("PagerFinishedRecovery");
} }
catch(Error &e) { catch(Error &e) {
@ -236,17 +242,17 @@ ACTOR Future<Void> housekeeper(IndirectShadowPager *pager) {
pager->freePhysicalPageID(prev->second); pager->freePhysicalPageID(prev->second);
} }
if(itr == pageVersionMap.end() || itr->first >= pager->oldestVersion) { if(itr == pageVersionMap.end() || itr->first >= pager->oldestVersion) {
//fprintf(stderr, "Updating oldest version for logical page %d: %d\n", pageID, pager->oldestVersion); //debug_printf("Updating oldest version for logical page %d: %d\n", pageID, pager->oldestVersion);
pager->logPageTableClear(pageID, 0, pager->oldestVersion); pager->logPageTableClear(pageID, 0, pager->oldestVersion);
if(itr != pageVersionMap.end() && itr->first > pager->oldestVersion) { if(itr != pageVersionMap.end() && itr->first > pager->oldestVersion) {
//fprintf(stderr, "Erasing pages to prev from pageVersionMap for %d (itr=%d, prev=%d)\n", pageID, itr->first, prev->first); //debug_printf("Erasing pages to prev from pageVersionMap for %d (itr=%d, prev=%d)\n", pageID, itr->first, prev->first);
prev->first = pager->oldestVersion; prev->first = pager->oldestVersion;
pager->logPageTableUpdate(pageID, pager->oldestVersion, prev->second); pager->logPageTableUpdate(pageID, pager->oldestVersion, prev->second);
itr = pageVersionMap.erase(pageVersionMap.begin(), prev); itr = pageVersionMap.erase(pageVersionMap.begin(), prev);
} }
else { else {
//fprintf(stderr, "Erasing pages to itr from pageVersionMap for %d (%d) (itr=%d, prev=%d)\n", pageID, itr == pageVersionMap.end(), itr==pageVersionMap.end() ? -1 : itr->first, prev->first); //debug_printf("Erasing pages to itr from pageVersionMap for %d (%d) (itr=%d, prev=%d)\n", pageID, itr == pageVersionMap.end(), itr==pageVersionMap.end() ? -1 : itr->first, prev->first);
itr = pageVersionMap.erase(pageVersionMap.begin(), itr); itr = pageVersionMap.erase(pageVersionMap.begin(), itr);
} }
} }
@ -337,13 +343,13 @@ void IndirectShadowPager::freeLogicalPage(LogicalPageID pageID, Version version)
} }
if(itr != pageVersionMap.end()) { if(itr != pageVersionMap.end()) {
//fprintf(stderr, "Clearing newest versions for logical page %d: %d\n", pageID, version); //debug_printf("Clearing newest versions for logical page %d: %d\n", pageID, version);
logPageTableClearToEnd(pageID, version); logPageTableClearToEnd(pageID, version);
pageVersionMap.erase(itr, pageVersionMap.end()); pageVersionMap.erase(itr, pageVersionMap.end());
} }
if(pageVersionMap.size() == 0) { if(pageVersionMap.size() == 0) {
fprintf(stderr, "Freeing logical page %d (freeLogicalPage)\n", pageID); debug_printf("Freeing logical page %d (freeLogicalPage)\n", pageID);
logicalFreeList.push_back(pageID); logicalFreeList.push_back(pageID);
} }
else if(pageVersionMap.back().second != PagerFile::INVALID_PAGE) { else if(pageVersionMap.back().second != PagerFile::INVALID_PAGE) {
@ -361,7 +367,7 @@ ACTOR Future<Void> waitAndFreePhysicalPageID(IndirectShadowPager *pager, Physica
// TODO: we need to be more careful about making sure the action that caused the page to be freed is durable before freeing the page // TODO: we need to be more careful about making sure the action that caused the page to be freed is durable before freeing the page
// Otherwise, the page could be rewritten prematurely. // Otherwise, the page could be rewritten prematurely.
void IndirectShadowPager::freePhysicalPageID(PhysicalPageID pageID) { void IndirectShadowPager::freePhysicalPageID(PhysicalPageID pageID) {
fprintf(stderr, "Freeing physical page: %u\n", pageID); debug_printf("Freeing physical page: %u\n", pageID);
auto itr = readCounts.find(pageID); auto itr = readCounts.find(pageID);
if(itr != readCounts.end()) { if(itr != readCounts.end()) {
operations.add(waitAndFreePhysicalPageID(this, pageID, itr->second.second.getFuture())); operations.add(waitAndFreePhysicalPageID(this, pageID, itr->second.second.getFuture()));
@ -520,7 +526,7 @@ ACTOR Future<Reference<const IPage>> getPageImpl(IndirectShadowPager *pager, Ref
--itr; --itr;
state PhysicalPageID physicalPageID = itr->second; state PhysicalPageID physicalPageID = itr->second;
fprintf(stderr, "Reading %d at v%d from %d\n", pageID, version, physicalPageID); debug_printf("Reading %d at v%d from %d\n", pageID, version, physicalPageID);
ASSERT(physicalPageID != PagerFile::INVALID_PAGE); ASSERT(physicalPageID != PagerFile::INVALID_PAGE);
++pager->readCounts[physicalPageID].first; ++pager->readCounts[physicalPageID].first;
@ -568,7 +574,7 @@ PageVersionMap::iterator IndirectShadowPager::pageVersionMapUpperBound(PageVersi
void IndirectShadowPager::freeLogicalPageID(LogicalPageID pageID) { void IndirectShadowPager::freeLogicalPageID(LogicalPageID pageID) {
if(pageID >= SERVER_KNOBS->PAGER_RESERVED_PAGES) { if(pageID >= SERVER_KNOBS->PAGER_RESERVED_PAGES) {
fprintf(stderr, "Freeing logical page: %u\n", pageID); debug_printf("Freeing logical page: %u\n", pageID);
logicalFreeList.push_back(pageID); logicalFreeList.push_back(pageID);
} }
} }
@ -669,7 +675,7 @@ ACTOR Future<Void> vacuumer(IndirectShadowPager *pager, PagerFile *pagerFile) {
if(!pagerFile->vacuumQueue.empty()) { if(!pagerFile->vacuumQueue.empty()) {
state PhysicalPageID lastUsedPage = pagerFile->vacuumQueue.rbegin()->first; state PhysicalPageID lastUsedPage = pagerFile->vacuumQueue.rbegin()->first;
PhysicalPageID lastFreePage = *pagerFile->freePages.rbegin(); PhysicalPageID lastFreePage = *pagerFile->freePages.rbegin();
//fprintf(stderr, "Vacuuming: evaluating (free list size=%u, lastFreePage=%u, lastUsedPage=%u, pagesAllocated=%u)\n", pagerFile->freePages.size(), lastFreePage, lastUsedPage, pagerFile->pagesAllocated); //debug_printf("Vacuuming: evaluating (free list size=%u, lastFreePage=%u, lastUsedPage=%u, pagesAllocated=%u)\n", pagerFile->freePages.size(), lastFreePage, lastUsedPage, pagerFile->pagesAllocated);
ASSERT(lastFreePage < pagerFile->pagesAllocated); ASSERT(lastFreePage < pagerFile->pagesAllocated);
ASSERT(lastUsedPage < pagerFile->pagesAllocated); ASSERT(lastUsedPage < pagerFile->pagesAllocated);
ASSERT(lastFreePage != lastUsedPage); ASSERT(lastFreePage != lastUsedPage);
@ -678,7 +684,7 @@ ACTOR Future<Void> vacuumer(IndirectShadowPager *pager, PagerFile *pagerFile) {
state std::pair<LogicalPageID, Version> logicalPageInfo = pagerFile->vacuumQueue[lastUsedPage]; state std::pair<LogicalPageID, Version> logicalPageInfo = pagerFile->vacuumQueue[lastUsedPage];
state PhysicalPageID newPage = pagerFile->allocatePage(logicalPageInfo.first, logicalPageInfo.second); state PhysicalPageID newPage = pagerFile->allocatePage(logicalPageInfo.first, logicalPageInfo.second);
fprintf(stderr, "Vacuuming: copying page %u to %u\n", lastUsedPage, newPage); debug_printf("Vacuuming: copying page %u to %u\n", lastUsedPage, newPage);
Void _ = wait(copyPage(pager, page, lastUsedPage, newPage)); Void _ = wait(copyPage(pager, page, lastUsedPage, newPage));
auto &pageVersionMap = pager->pageTable[logicalPageInfo.first]; auto &pageVersionMap = pager->pageTable[logicalPageInfo.first];
@ -703,11 +709,11 @@ ACTOR Future<Void> vacuumer(IndirectShadowPager *pager, PagerFile *pagerFile) {
pagesToErase = std::min<uint64_t>(pagerFile->freePages.size() - SERVER_KNOBS->FREE_PAGE_VACUUM_THRESHOLD + 1, pagerFile->pagesAllocated - firstFreePage); pagesToErase = std::min<uint64_t>(pagerFile->freePages.size() - SERVER_KNOBS->FREE_PAGE_VACUUM_THRESHOLD + 1, pagerFile->pagesAllocated - firstFreePage);
} }
//fprintf(stderr, "Vacuuming: got %u pages to erase (freePages=%u, pagesAllocated=%u, vacuumQueueEmpty=%u, minVacuumQueuePage=%u, firstFreePage=%u)\n", pagesToErase, pagerFile->freePages.size(), pagerFile->pagesAllocated, pagerFile->vacuumQueue.empty(), pagerFile->minVacuumQueuePage, firstFreePage); //debug_printf("Vacuuming: got %u pages to erase (freePages=%u, pagesAllocated=%u, vacuumQueueEmpty=%u, minVacuumQueuePage=%u, firstFreePage=%u)\n", pagesToErase, pagerFile->freePages.size(), pagerFile->pagesAllocated, pagerFile->vacuumQueue.empty(), pagerFile->minVacuumQueuePage, firstFreePage);
if(pagesToErase > 0) { if(pagesToErase > 0) {
PhysicalPageID eraseStartPage = pagerFile->pagesAllocated - pagesToErase; PhysicalPageID eraseStartPage = pagerFile->pagesAllocated - pagesToErase;
fprintf(stderr, "Vacuuming: truncating last %u pages starting at %u\n", pagesToErase, eraseStartPage); debug_printf("Vacuuming: truncating last %u pages starting at %u\n", pagesToErase, eraseStartPage);
ASSERT(pagesToErase <= pagerFile->pagesAllocated); ASSERT(pagesToErase <= pagerFile->pagesAllocated);
@ -751,7 +757,7 @@ PhysicalPageID PagerFile::allocatePage(LogicalPageID logicalPageID, Version vers
markPageAllocated(logicalPageID, version, allocatedPage); markPageAllocated(logicalPageID, version, allocatedPage);
fprintf(stderr, "Allocated physical page %u\n", allocatedPage); debug_printf("Allocated physical page %u\n", allocatedPage);
return allocatedPage; return allocatedPage;
} }
@ -813,7 +819,7 @@ bool PagerFile::canVacuum() {
|| minVacuumQueuePage >= pagesAllocated // We finished processing all pages in the vacuum queue || minVacuumQueuePage >= pagesAllocated // We finished processing all pages in the vacuum queue
|| !vacuumQueueReady) // Populating vacuum queue || !vacuumQueueReady) // Populating vacuum queue
{ {
//fprintf(stderr, "Vacuuming: waiting for vacuumable pages (free list size=%u, minVacuumQueuePage=%u, pages allocated=%u, vacuumQueueReady=%d)\n", freePages.size(), minVacuumQueuePage, pagesAllocated, vacuumQueueReady); //debug_printf("Vacuuming: waiting for vacuumable pages (free list size=%u, minVacuumQueuePage=%u, pages allocated=%u, vacuumQueueReady=%d)\n", freePages.size(), minVacuumQueuePage, pagesAllocated, vacuumQueueReady);
return false; return false;
} }

35
fdbserver/VersionedBTree.actor.cpp
View File

@ -32,7 +32,11 @@
#define INTERNAL_PAGES_HAVE_TUPLES 1 #define INTERNAL_PAGES_HAVE_TUPLES 1
#define debug_printf(...) #ifdef REDWOOD_DEBUG
#define debug_printf printf
#else
#define debug_printf(...)
#endif
struct SimpleFixedSizeMapRef { struct SimpleFixedSizeMapRef {
typedef std::vector<std::pair<std::string, std::string>> KVPairsT; typedef std::vector<std::pair<std::string, std::string>> KVPairsT;
@ -158,14 +162,15 @@ public:
} }
ACTOR static Future<Void> init(VersionedBTree *self) { ACTOR static Future<Void> init(VersionedBTree *self) {
// TODO: don't just create a new root, load the existing one self->m_root = 0;
Version latest = wait(self->m_pager->getLatestVersion()); state Version latest = wait(self->m_pager->getLatestVersion());
self->m_root = self->m_pager->allocateLogicalPage(); if(latest == 0) {
Version v = latest + 1; IPager *pager = self->m_pager;
IPager *pager = self->m_pager; self->writePage(self->m_root, FixedSizeMap::emptyPage(EPageFlags::IS_LEAF, [pager](){ return pager->newPageBuffer(); }), 1);
self->writePage(self->m_root, FixedSizeMap::emptyPage(EPageFlags::IS_LEAF, [pager](){ return pager->newPageBuffer(); }), v); self->m_pager->setLatestVersion(1);
self->m_pager->setLatestVersion(v); Void _ = wait(self->m_pager->commit());
Void _ = wait(self->m_pager->commit()); }
printf("latest %lld\n", latest);
return Void(); return Void();
} }
@ -563,9 +568,10 @@ private:
state KeyRef tupleKey = t.pack(); state KeyRef tupleKey = t.pack();
loop { loop {
debug_printf("findEqual: Reading page %d\n", pageNumber);
Reference<const IPage> rawPage = wait(self->m_pager->getPhysicalPage(pageNumber)); Reference<const IPage> rawPage = wait(self->m_pager->getPhysicalPage(pageNumber));
FixedSizeMap map = FixedSizeMap::decode(StringRef(rawPage->begin(), rawPage->size())); FixedSizeMap map = FixedSizeMap::decode(StringRef(rawPage->begin(), rawPage->size()));
//debug_printf("Read page %d @%lld: %s\n", pageNumber, self->m_version, map.toString().c_str()); debug_printf("Read page %d @%lld: %s\n", pageNumber, self->m_version, map.toString().c_str());
// Special case of empty page (which should only happen for root) // Special case of empty page (which should only happen for root)
if(map.entries.empty()) { if(map.entries.empty()) {
@ -620,7 +626,10 @@ TEST_CASE("/redwood/correctness/memory/set") {
else else
pager = createMemoryPager(); pager = createMemoryPager();
state VersionedBTree *btree = new VersionedBTree(pager, g_random->randomInt(50, 200)); state int pageSize = g_random->coinflip() ? -1 : g_random->randomInt(50, 200);
state VersionedBTree *btree = new VersionedBTree(pager, pageSize);
Void _ = wait(btree->init()); Void _ = wait(btree->init());
state std::map<std::pair<std::string, Version>, std::string> written; state std::map<std::pair<std::string, Version>, std::string> written;
@ -663,7 +672,7 @@ TEST_CASE("/redwood/correctness/memory/set") {
pager = new IndirectShadowPager("pagerfile"); pager = new IndirectShadowPager("pagerfile");
btree = new VersionedBTree(pager, g_random->randomInt(50, 200)); btree = new VersionedBTree(pager, pageSize);
Void _ = wait(btree->init()); Void _ = wait(btree->init());
Version lastVer = wait(btree->getLatestVersion()); Version lastVer = wait(btree->getLatestVersion());
@ -712,7 +721,7 @@ TEST_CASE("/redwood/performance/set") {
state int nodeCount = 100000; state int nodeCount = 100000;
state int maxChangesPerVersion = 5; state int maxChangesPerVersion = 5;
state int versions = 4; state int versions = 1000;
int maxKeySize = 100; int maxKeySize = 100;
int maxValueSize = 500; int maxValueSize = 500;