diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp index bf91fdecc9..4ad8ce6dc0 100644 --- a/FDBLibTLS/FDBLibTLSSession.cpp +++ b/FDBLibTLS/FDBLibTLSSession.cpp @@ -243,11 +243,11 @@ std::tuple FDBLibTLSSession::check_verify(Referenceroots); @@ -256,31 +256,31 @@ std::tuple FDBLibTLSSession::check_verify(Referencechain, 0); if ((subject = X509_get_subject_name(cert)) == NULL) { - reason = "FDBLibTLSCertSubjectError"; + reason = "Cert subject error"; goto err; } for (auto &pair: verify->subject_criteria) { if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { - reason = "FDBLibTLSCertSubjectMatchFailure"; + reason = "Cert subject match failure"; goto err; } } // Check issuer criteria. if ((issuer = X509_get_issuer_name(cert)) == NULL) { - reason = "FDBLibTLSCertIssuerError"; + reason = "Cert issuer error"; goto err; } for (auto &pair: verify->issuer_criteria) { if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { - reason = "FDBLibTLSCertIssuerMatchFailure"; + reason = "Cert issuer match failure"; goto err; } } @@ -288,12 +288,12 @@ std::tuple FDBLibTLSSession::check_verify(Referencechain, sk_X509_num(store_ctx->chain) - 1); if ((subject = X509_get_subject_name(cert)) == NULL) { - reason = "FDBLibTLSRootSubjectError"; + reason = "Root subject error"; goto err; } for (auto &pair: verify->root_criteria) { if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { - reason = "FDBLibTLSRootSubjectMatchFailure"; + reason = "Root subject match failure"; goto err; } } @@ -343,7 +343,7 @@ bool FDBLibTLSSession::verify_peer() { if (!rc) { // log the various failure reasons for (std::string reason : verify_failure_reasons) { - TraceEvent(reason.c_str(), uid).suppressFor(1.0); + TraceEvent("FDBLibTLSVerifyFailure", uid).detail("Reason", reason).suppressFor(1.0); } } diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index d4bb3bb93b..63a25896f6 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.1.5.pkg `_ +* `FoundationDB-6.1.6.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.1.5-1_amd64.deb `_ -* `foundationdb-server-6.1.5-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.1.6-1_amd64.deb `_ +* `foundationdb-server-6.1.6-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.1.5-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.1.5-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.1.6-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.1.6-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.1.5-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.1.5-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.1.6-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.1.6-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.1.5-x64.msi `_ +* `foundationdb-6.1.6-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package: -* `foundationdb-6.1.5.tar.gz `_ +* `foundationdb-6.1.6.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.1.5.gem `_ +* `fdb-6.1.6.gem `_ Java 8+ ------- -* `fdb-java-6.1.5.jar `_ -* `fdb-java-6.1.5-javadoc.jar `_ +* `fdb-java-6.1.6.jar `_ +* `fdb-java-6.1.6-javadoc.jar `_ Go 1.1+ ------- diff --git a/documentation/sphinx/source/old-release-notes/release-notes-610.rst b/documentation/sphinx/source/old-release-notes/release-notes-610.rst index d8c309d61b..f4f1aa123f 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-610.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-610.rst @@ -2,7 +2,7 @@ Release Notes ############# -6.1.5 +6.1.6 ===== Features @@ -126,6 +126,7 @@ Fixes only impacting 6.1.0+ * The transaction log spill-by-reference policy could read too much data from disk. [6.1.5] `(PR #1527) `_ * Memory tracking trace events could cause the program to crash when called from inside a trace event. [6.1.5] `(PR #1541) `_ * TLogs will replace a large file with an empty file rather than doing a large truncate operation. [6.1.5] `(PR #1545) `_ +* Fix PR #1545 to work on Windows and Linux. [6.1.6] `(PR #1556) `_ Earlier release notes --------------------- diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h index 362b4119fe..ce49df0113 100644 --- a/fdbrpc/AsyncFileKAIO.actor.h +++ b/fdbrpc/AsyncFileKAIO.actor.h @@ -265,7 +265,7 @@ public: result = fallocate( fd, 0, 0, size); if (result != 0) { int fallocateErrCode = errno; - TraceEvent("AsyncFileKAIOAllocateError").detail("Fd",fd).detail("Filename", filename).GetLastError(); + TraceEvent("AsyncFileKAIOAllocateError").detail("Fd",fd).detail("Filename", filename).detail("Size", size).GetLastError(); if ( fallocateErrCode == EOPNOTSUPP ) { // Mark fallocate as unsupported. Try again with truncate. ctx.fallocateSupported = false; diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 518f9c8338..be9a0bd4e6 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -603,11 +603,16 @@ private: if (randLog) fprintf( randLog, "SFT1 %s %s %s %" PRId64 "\n", self->dbgId.shortString().c_str(), self->filename.c_str(), opId.shortString().c_str(), size ); + if (size == 0) { + // KAIO will return EINVAL, as len==0 is an error. + throw io_error(); + } + if(self->delayOnWrite) wait( waitUntilDiskReady( self->diskParameters, 0 ) ); if( _chsize( self->h, (long) size ) == -1 ) { - TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6); + TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6).detail("Filename", self->filename).detail("Size", size).detail("Fd", self->h).GetLastError(); throw io_error(); } diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp index 7097698710..3f5e2779cf 100644 --- a/fdbserver/DiskQueue.actor.cpp +++ b/fdbserver/DiskQueue.actor.cpp @@ -164,7 +164,7 @@ public: readyToPush(Void()), fileSizeWarningLimit(fileSizeWarningLimit), lastCommit(Void()), isFirstCommit(true) { if (BUGGIFY) - fileExtensionBytes = 1<<10 * g_random->randomSkewedUInt32( 1, 40<<10 ); + fileExtensionBytes = _PAGE_SIZE * g_random->randomSkewedUInt32( 1, 10<<10 ); if (BUGGIFY) fileShrinkBytes = _PAGE_SIZE * g_random->randomSkewedUInt32( 1, 10<<10 ); files[0].dbgFilename = filename(0); @@ -283,21 +283,29 @@ public: TraceEvent("DiskQueueReplaceTruncateEnded").detail("Filename", file->getFilename()); } +#if defined(_WIN32) + ACTOR static Future> replaceFile(Reference toReplace) { + // Windows doesn't support a rename over an open file. + wait( toReplace->truncate(4<<10) ); + return toReplace; + } +#else ACTOR static Future> replaceFile(Reference toReplace) { incrementalTruncate( toReplace ); - Reference _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK, 0 ) ); + Reference _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK, 0600 ) ); state Reference replacement = _replacement; wait( replacement->sync() ); return replacement; } +#endif - Future push(StringRef pageData, vector>* toSync) { + Future> push(StringRef pageData, vector>* toSync) { return push( this, pageData, toSync ); } - ACTOR static Future push(RawDiskQueue_TwoFiles* self, StringRef pageData, vector>* toSync) { + ACTOR static Future> push(RawDiskQueue_TwoFiles* self, StringRef pageData, vector>* toSync) { // Write the given data to the queue files, swapping or extending them if necessary. // Don't do any syncs, but push the modified file(s) onto toSync. ASSERT( self->readingFile == 2 ); @@ -325,21 +333,27 @@ public: std::swap(self->firstPages[0], self->firstPages[1]); self->files[1].popped = 0; self->writingPos = 0; + *self->firstPages[1] = *(const Page*)pageData.begin(); const int64_t activeDataVolume = pageCeiling(self->files[0].size - self->files[0].popped + self->fileExtensionBytes + self->fileShrinkBytes); - const int64_t desiredMaxFileSize = std::max( activeDataVolume, SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES * 2 ); - if (self->files[1].size > desiredMaxFileSize) { + const int64_t desiredMaxFileSize = pageCeiling( std::max( activeDataVolume, SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES * 2 ) ); + const bool frivolouslyTruncate = BUGGIFY_WITH_PROB(0.1); + if (self->files[1].size > desiredMaxFileSize || frivolouslyTruncate) { // Either shrink self->files[1] to the size of self->files[0], or chop off fileShrinkBytes - int64_t maxShrink = std::max( pageFloor(self->files[1].size - desiredMaxFileSize), self->fileShrinkBytes ); - if (maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES > - SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) { + int64_t maxShrink = pageFloor( std::max( self->files[1].size - desiredMaxFileSize, self->fileShrinkBytes ) ); + if ((maxShrink > SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_BYTES) || + (frivolouslyTruncate && g_random->random01() < 0.3)) { TEST(true); // Replacing DiskQueue file TraceEvent("DiskQueueReplaceFile", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", self->files[1].size).detail("ElidedTruncateSize", maxShrink); Reference newFile = wait( replaceFile(self->files[1].f) ); self->files[1].setFile(newFile); - self->files[1].size = 0; + waitfor.push_back( self->files[1].f->truncate( self->fileExtensionBytes ) ); + self->files[1].size = self->fileExtensionBytes; } else { - self->files[1].size -= maxShrink; + const int64_t startingSize = self->files[1].size; + self->files[1].size -= std::min(maxShrink, self->files[1].size); + self->files[1].size = std::max(self->files[1].size, self->fileExtensionBytes); + TraceEvent("DiskQueueTruncate", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", startingSize).detail("NewFileSize", self->files[1].size); waitfor.push_back( self->files[1].f->truncate( self->files[1].size ) ); } } @@ -355,9 +369,8 @@ public: TraceEvent(SevWarnAlways, "DiskQueueFileTooLarge", self->dbgid).suppressFor(1.0).detail("Filename", self->filename(1)).detail("Size", self->files[1].size); } } - } - - if (self->writingPos == 0) { + } else if (self->writingPos == 0) { + // If this is the first write to a brand new disk queue file. *self->firstPages[1] = *(const Page*)pageData.begin(); } @@ -368,8 +381,7 @@ public: waitfor.push_back( self->files[1].f->write( pageData.begin(), pageData.size(), self->writingPos ) ); self->writingPos += pageData.size(); - wait( waitForAll(waitfor) ); - return Void(); + return waitForAll(waitfor); } ACTOR static UNCANCELLABLE Future pushAndCommit(RawDiskQueue_TwoFiles* self, StringRef pageData, StringBuffer* pageMem, uint64_t poppedPages) { @@ -396,11 +408,11 @@ public: TEST( pageData.size() > sizeof(Page) ); // push more than one page of data - Future pushed = self->push( pageData, &syncFiles ); + Future pushed = wait( self->push( pageData, &syncFiles ) ); pushing.send(Void()); - wait( pushed ); ASSERT( syncFiles.size() >= 1 && syncFiles.size() <= 2 ); TEST(2==syncFiles.size()); // push spans both files + wait( pushed ); delete pageMem; pageMem = 0; diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index 93e2e6e63a..73840caa6d 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -75,7 +75,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) { init( TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH, 16<<10 ); if ( randomize && BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH = 500; init( DISK_QUEUE_FILE_EXTENSION_BYTES, 10<<20 ); // BUGGIFYd per file within the DiskQueue init( DISK_QUEUE_FILE_SHRINK_BYTES, 100<<20 ); // BUGGIFYd per file within the DiskQueue - init( DISK_QUEUE_MAX_TRUNCATE_EXTENTS, 1<<10 ); if ( randomize && BUGGIFY ) DISK_QUEUE_MAX_TRUNCATE_EXTENTS = 0; + init( DISK_QUEUE_MAX_TRUNCATE_BYTES, 2<<30 ); if ( randomize && BUGGIFY ) DISK_QUEUE_MAX_TRUNCATE_BYTES = 0; init( TLOG_DEGRADED_DELAY_COUNT, 5 ); init( TLOG_DEGRADED_DURATION, 5.0 ); diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h index 6287fd1911..e7a43d0a2f 100644 --- a/fdbserver/Knobs.h +++ b/fdbserver/Knobs.h @@ -79,7 +79,7 @@ public: int64_t TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH; int64_t DISK_QUEUE_FILE_EXTENSION_BYTES; // When we grow the disk queue, by how many bytes should it grow? int64_t DISK_QUEUE_FILE_SHRINK_BYTES; // When we shrink the disk queue, by how many bytes should it shrink? - int DISK_QUEUE_MAX_TRUNCATE_EXTENTS; + int DISK_QUEUE_MAX_TRUNCATE_BYTES; // A truncate larger than this will cause the file to be replaced instead. int TLOG_DEGRADED_DELAY_COUNT; double TLOG_DEGRADED_DURATION; diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 9480cc2036..9947b6138a 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -32,7 +32,7 @@