wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/apple/foundationdb into jfu-incremental-backup-only

This commit is contained in:
Jon Fu 2020-08-19 16:36:47 -04:00
parent 0bc2e4f58d 4867bad46e
commit ae999aa118
208 changed files with 6447 additions and 15055 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ACKNOWLEDGEMENTS
View File

@ -85,28 +85,6 @@ Steve Dekorte (libcoroutine)
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Jean-loup Gailly, Mark Adler (zlib)
Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Jean-loup Gailly Mark Adler
jloup@gzip.org madler@alumni.caltech.edu
The Go Authors (Go Tools)
Copyright (c) 2009 The Go Authors. All rights reserved.

5
CMakeLists.txt
View File

@ -177,6 +177,11 @@ else()
include(CPack)
endif()
set(BUILD_FLOWBENCH OFF CACHE BOOL "Build microbenchmark program (builds google microbenchmark dependency)")
if(BUILD_FLOWBENCH)
add_subdirectory(flowbench)
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
add_link_options(-lexecinfo)
endif()

19
bindings/c/test/mako/mako.c
View File

@ -999,12 +999,12 @@ failExit:
int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pid_t* pid_main) {
int i;
pthread_t network_thread; /* handle for thread which invoked fdb_run_network() */
pthread_t* worker_threads;
pthread_t* worker_threads = NULL;
#if FDB_API_VERSION < 610
FDBCluster* cluster;
#endif
process_info_t process;
thread_args_t* thread_args;
thread_args_t* thread_args = NULL;
int rc;
fdb_error_t err;
@ -1017,7 +1017,11 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* Everything starts from here */
err = fdb_select_api_version(args->api_version);
check_fdb_error(err);
if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
}
/* enable flatbuffers if specified */
if (args->flatbuffers) {
@ -1065,7 +1069,11 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* Network thread must be setup before doing anything */
fprintf(debugme, "DEBUG: fdb_setup_network\n");
err = fdb_setup_network();
check_fdb_error(err);
if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
}
/* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n");
@ -2067,7 +2075,7 @@ int main(int argc, char* argv[]) {
int rc;
mako_args_t args;
int p;
pid_t* worker_pids;
pid_t* worker_pids = NULL;
proc_type_t proc_type = proc_master;
int worker_id;
pid_t pid;
@ -2117,6 +2125,7 @@ int main(int argc, char* argv[]) {
/* allocate */
shmsize = sizeof(mako_shmhdr_t) + (sizeof(mako_stats_t) * args.num_processes * args.num_threads);
if (ftruncate(shmfd, shmsize) < 0) {
shm = MAP_FAILED;
fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize);
goto failExit;
}

6
bindings/flow/tester/Tester.actor.cpp
View File

@ -1365,12 +1365,12 @@ const char* StartThreadFunc::name = "START_THREAD";
REGISTER_INSTRUCTION_FUNC(StartThreadFunc);
ACTOR template <class Function>
Future<decltype(fake<Function>()(Reference<ReadTransaction>()).getValue())> read(Reference<Database> db,
Function func) {
Future<decltype(std::declval<Function>()(Reference<ReadTransaction>()).getValue())> read(Reference<Database> db,
Function func) {
state Reference<ReadTransaction> tr = db->createTransaction();
loop {
try {
state decltype(fake<Function>()(Reference<ReadTransaction>()).getValue()) result = wait(func(tr));
state decltype(std::declval<Function>()(Reference<ReadTransaction>()).getValue()) result = wait(func(tr));
return result;
} catch (Error& e) {
wait(tr->onError(e));

20
bindings/flow/tester/Tester.actor.h
View File

@ -27,6 +27,8 @@
#pragma once
#include <utility>
#include "flow/IDispatched.h"
#include "bindings/flow/fdb_flow.h"
#include "bindings/flow/IDirectory.h"
@ -57,7 +59,7 @@ struct FlowTesterStack {
void push(Future<Standalone<StringRef>> value) {
data.push_back(StackItem(index, value));
}
void push(Standalone<StringRef> value) {
push(Future<Standalone<StringRef>>(value));
}
@ -86,10 +88,10 @@ struct FlowTesterStack {
items.push_back(data.back());
data.pop_back();
count--;
}
}
return items;
}
Future<std::vector<FDB::Tuple>> waitAndPop(int count);
Future<FDB::Tuple> waitAndPop();
@ -106,7 +108,7 @@ struct FlowTesterStack {
struct InstructionData : public ReferenceCounted<InstructionData> {
bool isDatabase;
bool isSnapshot;
bool isSnapshot;
StringRef instruction;
Reference<FDB::Transaction> tr;
@ -153,7 +155,7 @@ struct DirectoryOrSubspace {
return "DirectorySubspace";
}
else if(directory.present()) {
return "IDirectory";
return "IDirectory";
}
else if(subspace.present()) {
return "Subspace";
@ -169,10 +171,10 @@ struct DirectoryTesterData {
int directoryListIndex;
int directoryErrorIndex;
Reference<FDB::IDirectory> directory() {
Reference<FDB::IDirectory> directory() {
ASSERT(directoryListIndex < directoryList.size());
ASSERT(directoryList[directoryListIndex].directory.present());
return directoryList[directoryListIndex].directory.get();
return directoryList[directoryListIndex].directory.get();
}
FDB::Subspace* subspace() {
@ -220,10 +222,10 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::string tupleToString(FDB::Tuple const& tuple);
ACTOR template <class F>
Future<decltype(fake<F>()().getValue())> executeMutation(Reference<InstructionData> instruction, F func) {
Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<InstructionData> instruction, F func) {
loop {
try {
state decltype(fake<F>()().getValue()) result = wait(func());
state decltype(std::declval<F>()().getValue()) result = wait(func());
if(instruction->isDatabase) {
wait(instruction->tr->commit());
}

6
bindings/go/src/fdb/transaction.go
View File

@ -320,6 +320,12 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur
// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
// stored in the given range.
// Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
// algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
// be sampled and the more accurate its sampled size would be. And due to
// that reason it is recommended to use this API to query against large ranges for accuracy considerations.
// For a rough reference, if the returned size is larger than 3MB, one can consider the size to be
// accurate.
func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
beginKey, endKey := r.FDBRangeKeys()
return t.getEstimatedRangeSizeBytes(

3
bindings/java/JavaWorkload.cpp
View File

@ -75,6 +75,9 @@ void printTrace(JNIEnv* env, jclass, jlong logger, jint severity, jstring messag
sev = FDBSeverity::Warn;
} else if (severity < 40) {
sev = FDBSeverity::WarnAlways;
} else {
assert(false);
std::abort();
}
log->trace(sev, msg, detailsMap);
if (isCopy) {

19
bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
View File

@ -48,7 +48,7 @@ public interface ReadTransaction extends ReadTransactionContext {
* whether read conflict ranges are omitted for any reads done through this {@code ReadTransaction}.
* <br>
* For more information about how to use snapshot reads correctly, see
* <a href="/foundationdb/developer-guide.html#using-snapshot-reads" target="_blank">Using snapshot reads</a>.
* <a href="/foundationdb/developer-guide.html#snapshot-reads" target="_blank">Using snapshot reads</a>.
*
* @return whether this is a snapshot view of the database with relaxed isolation properties
* @see #snapshot()
@ -58,11 +58,11 @@ public interface ReadTransaction extends ReadTransactionContext {
/**
* Return a special-purpose, read-only view of the database. Reads done through this interface are known as "snapshot reads".
* Snapshot reads selectively relax FoundationDB's isolation property, reducing
* <a href="/foundationdb/developer-guide.html#transaction-conflicts" target="_blank">Transaction conflicts</a>
* <a href="/foundationdb/developer-guide.html#conflict-ranges" target="_blank">Transaction conflicts</a>
* but making reasoning about concurrency harder.<br>
* <br>
* For more information about how to use snapshot reads correctly, see
* <a href="/foundationdb/developer-guide.html#using-snapshot-reads" target="_blank">Using snapshot reads</a>.
* <a href="/foundationdb/developer-guide.html#snapshot-reads" target="_blank">Using snapshot reads</a>.
*
* @return a read-only view of this {@code ReadTransaction} with relaxed isolation properties
*/
@ -427,6 +427,12 @@ public interface ReadTransaction extends ReadTransactionContext {
/**
* Gets an estimate for the number of bytes stored in the given range.
* Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
* algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
* be sampled and the more accurate its sampled size would be. And due to
* that reason it is recommended to use this API to query against large ranges for accuracy considerations.
* For a rough reference, if the returned size is larger than 3MB, one can consider the size to be
* accurate.
*
* @param begin the beginning of the range (inclusive)
* @param end the end of the range (exclusive)
@ -437,7 +443,12 @@ public interface ReadTransaction extends ReadTransactionContext {
/**
* Gets an estimate for the number of bytes stored in the given range.
*
* Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
* algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
* be sampled and the more accurate its sampled size would be. And due to
* that reason it is recommended to use this API to query against large ranges for accuracy considerations.
* For a rough reference, if the returned size is larger than 3MB, one can consider the size to be
* accurate.
* @param range the range of the keys
*
* @return a handle to access the results of the asynchronous call

18
bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java
View File

@ -30,7 +30,7 @@ import sun.misc.Unsafe;
/**
* Utility code to do optimized byte-array comparison.
* This is borrowed and slightly modified from Guava's {@link UnsignedBytes}
* This is borrowed and slightly modified from Guava's UnsignedBytes
* class to be able to compare arrays that start at non-zero offsets.
*/
abstract class FastByteComparisons {
@ -38,14 +38,14 @@ abstract class FastByteComparisons {
private static final int UNSIGNED_MASK = 0xFF;
/**
* Lexicographically compare two byte arrays.
*
*
* @param buffer1 left operand, expected to not be null
* @param buffer2 right operand, expected to not be null
* @param offset1 Where to start comparing in the left buffer, expected to be >= 0
* @param offset2 Where to start comparing in the right buffer, expected to be >= 0
* @param length1 How much to compare from the left buffer, expected to be >= 0
* @param length2 How much to compare from the right buffer, expected to be >= 0
* @return 0 if equal, < 0 if left is less than right, etc.
* @param offset1 Where to start comparing in the left buffer, expected to be &gt;= 0
* @param offset2 Where to start comparing in the right buffer, expected to be &gt;= 0
* @param length1 How much to compare from the left buffer, expected to be &gt;= 0
* @param length2 How much to compare from the right buffer, expected to be &gt;= 0
* @return 0 if equal, &lt; 0 if left is less than right, etc.
*/
public static int compareTo(byte[] buffer1, int offset1, int length1,
byte[] buffer2, int offset2, int length2) {
@ -59,7 +59,7 @@ abstract class FastByteComparisons {
interface Comparer<T> extends Comparator<T> {
/**
* Lexicographically compare two byte arrays.
*
*
* @param buffer1 left operand
* @param buffer2 right operand
* @param offset1 Where to start comparing in the left buffer
@ -291,4 +291,4 @@ abstract class FastByteComparisons {
}
}
}
}
}

2
cmake/CompileBoost.cmake
View File

@ -22,5 +22,5 @@ else()
add_library(boost_target INTERFACE)
add_dependencies(boost_target boostProject)
target_include_directories(boost_target INTERFACE ${BOOST_INCLUDE_DIR})
target_include_directories(boost_target SYSTEM INTERFACE ${BOOST_INCLUDE_DIR})
endif()

2
cmake/CompileRocksDB.cmake
View File

@ -10,6 +10,7 @@ if (RocksDB_FOUND)
DOWNLOAD_COMMAND ""
CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DWITH_GFLAGS=OFF
-DWITH_TESTS=OFF
@ -36,6 +37,7 @@ else()
URL_HASH SHA256=d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee
CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DWITH_GFLAGS=OFF
-DWITH_TESTS=OFF

29
cmake/ConfigureCompiler.cmake
View File

@ -215,6 +215,9 @@ else()
if (USE_AVX512F)
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
add_compile_options(-mavx512f)
elseif(USE_VALGRIND)
message(STATUS "USE_VALGRIND=ON make USE_AVX OFF to satisfy valgrind analysis requirement")
set(USE_AVX512F OFF)
else()
message(STATUS "USE_AVX512F is supported on x86 or x86_64 only")
set(USE_AVX512F OFF)
@ -224,6 +227,9 @@ else()
if (USE_AVX)
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^x86")
add_compile_options(-mavx)
elseif(USE_VALGRIND)
message(STATUS "USE_VALGRIND=ON make USE_AVX OFF to satisfy valgrind analysis requirement")
set(USE_AVX OFF)
else()
message(STATUS "USE_AVX is supported on x86 or x86_64 only")
set(USE_AVX OFF)
@ -266,18 +272,31 @@ else()
-Wno-unknown-attributes)
endif()
add_compile_options(
-Wno-unknown-warning-option
-Wno-dangling-else
-Wno-sign-compare
-Wall -Wextra
# Here's the current set of warnings we need to explicitly disable to compile warning-free with clang 10
-Wno-comment
-Wno-unknown-pragmas
-Wno-dangling-else
-Wno-delete-non-virtual-dtor
-Wno-format
-Wno-mismatched-tags
-Wno-missing-field-initializers
-Wno-overloaded-virtual
-Wno-reorder
-Wno-reorder-ctor
-Wno-sign-compare
-Wno-tautological-pointer-compare
-Wno-undefined-var-template
-Wno-tautological-pointer-compare
-Wno-format
-Wredundant-move
-Wpessimizing-move
-Woverloaded-virtual
-Wno-unknown-pragmas
-Wno-unknown-warning-option
-Wno-unused-function
-Wno-unused-local-typedef
-Wno-unused-parameter
-Wno-unused-value
-Wno-self-assign
)
if (USE_CCACHE)
add_compile_options(

6
contrib/TestHarness/CMakeLists.txt
View File

@ -1,12 +1,14 @@
set(SRCS
Program.cs
${CMAKE_CURRENT_BINARY_DIR}/Program.cs
Properties/AssemblyInfo.cs)
set(TEST_HARNESS_REFERENCES
"-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml,${TraceLogHelperDll}")
"-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml,System.Runtime.Serialization,${TraceLogHelperDll}")
set(out_file ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Program.cs.cmake ${CMAKE_CURRENT_BINARY_DIR}/Program.cs)
add_custom_command(OUTPUT ${out_file}
COMMAND ${MCS_EXECUTABLE} ARGS ${TEST_HARNESS_REFERENCES} ${SRCS} "-target:exe" "-out:${out_file}"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

76
contrib/TestHarness/Program.cs → contrib/TestHarness/Program.cs.cmake
View File

@ -29,6 +29,7 @@ using System.Diagnostics;
using System.ComponentModel;
using System.Runtime.InteropServices;
using System.Xml;
using System.Runtime.Serialization.Json;
namespace SummarizeTest
{
@ -153,6 +154,10 @@ namespace SummarizeTest
throw;
}
}
else if (args[0] == "version")
{
return VersionInfo();
}
return UsageMessage();
}
@ -261,7 +266,7 @@ namespace SummarizeTest
testFile = random.Choice(uniqueFiles);
string oldBinaryVersionLowerBound = "0.0.0";
string lastFolderName = Path.GetFileName(Path.GetDirectoryName(testFile));
if (lastFolderName.Contains("from_")) // Only perform upgrade tests from certain versions
if (lastFolderName.Contains("from_") || lastFolderName.Contains("to_")) // Only perform upgrade/downgrade tests from certain versions
{
oldBinaryVersionLowerBound = lastFolderName.Split('_').Last();
}
@ -295,14 +300,17 @@ namespace SummarizeTest
if (testDir.EndsWith("restarting"))
{
bool isDowngrade = Path.GetFileName(Path.GetDirectoryName(testFile)).Contains("to_");
string firstServerName = isDowngrade ? fdbserverName : oldServerName;
string secondServerName = isDowngrade ? oldServerName : fdbserverName;
int expectedUnseed = -1;
int unseed;
string uid = Guid.NewGuid().ToString();
bool useNewPlugin = oldServerName == fdbserverName || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
result = RunTest(oldServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout);
bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0");
result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout);
if (result == 0)
{
result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout);
result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout);
}
}
else
@ -359,20 +367,22 @@ namespace SummarizeTest
{
ErrorOutputListener errorListener = new ErrorOutputListener();
process.StartInfo.UseShellExecute = false;
string tlsPluginArg = "";
if (tlsPluginFile.Length > 0) {
process.StartInfo.EnvironmentVariables["FDB_TLS_PLUGIN"] = tlsPluginFile;
tlsPluginArg = "--tls_plugin=" + tlsPluginFile;
}
process.StartInfo.RedirectStandardOutput = true;
var args = "";
if (willRestart && oldBinaryName.EndsWith("alpha6"))
{
args = string.Format("-Rs 1000000000 -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash",
IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile);
args = string.Format("-Rs 1000000000 -r simulation {0} -s {1} -f \"{2}\" -b {3} {4} --crash",
IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg);
}
else
{
args = string.Format("-Rs 1GB -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash",
IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile);
args = string.Format("-Rs 1GB -r simulation {0} -s {1} -f \"{2}\" -b {3} {4} --crash",
IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg);
}
if (restarting) args = args + " --restarting";
if (useValgrind && !willRestart)
@ -477,7 +487,7 @@ namespace SummarizeTest
memCheckThread.Join();
consoleThread.Join();
var traceFiles = Directory.GetFiles(tempPath, "trace*.xml");
var traceFiles = Directory.GetFiles(tempPath, "trace*.*").Where(s => s.EndsWith(".xml") || s.EndsWith(".json")).ToArray();
if (traceFiles.Length == 0)
{
if (!traceToStdout)
@ -658,6 +668,10 @@ namespace SummarizeTest
return whats.ToArray();
}
delegate IEnumerable<Magnesium.Event> parseDelegate(System.IO.Stream stream, string file,
bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue,
double samplingFactor = 1.0);
static int Summarize(string[] traceFiles, string summaryFileName,
string errorFileName, bool? killed, List<string> outputErrors, int? exitCode, long? peakMemory,
string uid, string valgrindOutputFileName, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError,
@ -689,7 +703,12 @@ namespace SummarizeTest
{
try
{
foreach (var ev in Magnesium.XmlParser.Parse(traceFile, traceFileName))
parseDelegate parse;
if (traceFileName.EndsWith(".json"))
parse = Magnesium.JsonParser.Parse;
else
parse = Magnesium.XmlParser.Parse;
foreach (var ev in parse(traceFile, traceFileName))
{
Magnesium.Severity newSeverity;
if (severityMap.TryGetValue(new KeyValuePair<string, Magnesium.Severity>(ev.Type, ev.Severity), out newSeverity))
@ -1089,10 +1108,20 @@ namespace SummarizeTest
private static void AppendToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, bool shouldLock = true)
{
bool useXml = true;
if (summaryFileName != null && summaryFileName.EndsWith(".json")) {
useXml = false;
}
if (traceToStdout)
{
using (var wr = System.Xml.XmlWriter.Create(Console.OpenStandardOutput(), new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true, Encoding = new System.Text.UTF8Encoding(false) }))
xout.WriteTo(wr);
if (useXml) {
using (var wr = System.Xml.XmlWriter.Create(Console.OpenStandardOutput(), new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true, Encoding = new System.Text.UTF8Encoding(false) }))
xout.WriteTo(wr);
} else {
using (var wr = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonWriter(Console.OpenStandardOutput()))
xout.WriteTo(wr);
}
Console.WriteLine();
return;
}
@ -1103,7 +1132,6 @@ namespace SummarizeTest
takeLock(summaryFileName);
try
{
using (var f = System.IO.File.Open(summaryFileName, System.IO.FileMode.Append, System.IO.FileAccess.Write))
{
if (f.Length == 0)
@ -1111,8 +1139,13 @@ namespace SummarizeTest
byte[] bytes = Encoding.UTF8.GetBytes("<Trace>");
f.Write(bytes, 0, bytes.Length);
}
using (var wr = System.Xml.XmlWriter.Create(f, new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true }))
xout.Save(wr);
if (useXml) {
using (var wr = System.Xml.XmlWriter.Create(f, new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true }))
xout.Save(wr);
} else {
using (var wr = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonWriter(f))
xout.WriteTo(wr);
}
var endl = Encoding.UTF8.GetBytes(Environment.NewLine);
f.Write(endl, 0, endl.Length);
}
@ -1123,6 +1156,7 @@ namespace SummarizeTest
releaseLock(summaryFileName);
}
}
private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null,
int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null)
{
@ -1492,6 +1526,16 @@ namespace SummarizeTest
}
}
private static int VersionInfo()
{
Console.WriteLine("Version: 1.02");
Console.WriteLine("FDB Project Ver: " + "${CMAKE_PROJECT_VERSION}");
Console.WriteLine("FDB Version: " + "${CMAKE_PROJECT_VERSION_MAJOR}" + "." + "${CMAKE_PROJECT_VERSION_MINOR}");
Console.WriteLine("Source Version: " + "${CURRENT_GIT_VERSION}");
return 1;
}
private static int UsageMessage()
{
Console.WriteLine("Usage:");
@ -1502,7 +1546,7 @@ namespace SummarizeTest
Console.WriteLine(" TestHarness remote [queue folder] [root foundation folder] [duration in hours] [amount of tests] [all/fast/<test_path>] [scope]");
Console.WriteLine(" TestHarness extract-errors [summary-file] [error-summary-file]");
Console.WriteLine(" TestHarness joshua-run <useValgrind> <maxTries>");
Console.WriteLine("Version: 1.01");
VersionInfo();
return 1;
}
}

4
contrib/TraceLogHelper/JsonParser.cs
View File

@ -51,7 +51,7 @@ namespace Magnesium
}
catch (Exception e)
{
throw new Exception(string.Format("Failed to parse {0}", root), e);
throw new Exception(string.Format("Failed to parse JSON {0}", root), e);
}
if (ev != null) yield return ev;
}
@ -81,7 +81,7 @@ namespace Magnesium
DDetails = xEvent.Elements()
.Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime"))
.ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value),
original = keepOriginalElement ? xEvent : null,
original = keepOriginalElement ? xEvent : null
};
}

2
contrib/TraceLogHelper/XmlParser.cs
View File

@ -53,7 +53,7 @@ namespace Magnesium
}
catch (Exception e)
{
throw new Exception(string.Format("Failed to parse {0}", xev), e);
throw new Exception(string.Format("Failed to parse XML {0}", xev), e);
}
if (ev != null) yield return ev;
}

52
contrib/fdbcstat/README.md Normal file
View File

@ -0,0 +1,52 @@
# fdbcstat
`fdbcstat` is a FoundationDB client monitoring tool which collects and displays transaction operation statistics inside the C API library (`libfdb_c.so`).
## How it works
`fdbcstat` utilizes [eBPF/bcc](https://github.com/iovisor/bcc) to attach to `libfdb_c.so` shared library and insert special instructions to collect statistics in several common `fdb_transaction_*` calls, then it periodically displays the aggregated statistics.
## How to use
### Syntax
`fdbcstat <full path to libfdb_c.so> <options...>`
### Options
- `-p` or `--pid` : Only capture statistics for the functions called by the specified process
- `-i` or `--interval` : Specify the time interval in seconds between 2 outputs (Default: 1)
- `-d` or `--duration` : Specify the total duration in seconds `fdbcstats` will run (Default: Unset / Forever)
- `-f` or `--functions` : Specify the comma-separated list of functions to monitor (Default: Unset / All supported functions)
### Supported Functions
- get
- get_range
- get_read_version
- set
- clear
- clear_range
- commit
### Examples
##### Collect all statistics and display every second
`fdbcstat /usr/lib64/libfdb_c.so`
##### Collect all statistics for PID 12345 for 60 seconds with 10 second interval
`fdbcstat /usr/lib64/libfdb_c.so -p 12345 -d 60 -i 10`
##### Collect statitics only for get and commit
`fdbcstat /usr/lib64/libfdb_c.so -f get,commit`
## Output Format
Each line contains multiple fields. The first field is the timestamp. Other fields are the statistics for each operation. Each operation field contains the following statistics in a slash (/) separated format.
- Function
- Number of calls per second
- Average latency in microseconds (us)
- Maximum latency in microseconds (us)
**Note**: The latency is computed as the time difference between the start time and the end time of the `fdb_transaction_*` function call except for `get`, `get_range`, `get_read_version` and `commit`. For those 4 functions, the latency is the time difference between the start time of the function and the end time of the following `fdb_future_block_until_ready` call.
## Sample Output
```
...
15:05:31 clear/22426/2/34 commit/18290/859/15977 get/56230/1110/12748 get_range/14141/23/75 set/6276/3/19
15:05:41 clear/24147/2/38 commit/18259/894/44259 get/57978/1098/15636 get_range/13171/23/90 set/6564/3/15
15:05:51 clear/21287/2/34 commit/18386/876/17824 get/58318/1106/30539 get_range/13018/23/68 set/6559/3/13
...
```

304
contrib/fdbcstat/fdbcstat Executable file
View File

@ -0,0 +1,304 @@
#!/usr/bin/env python
from __future__ import print_function
from bcc import BPF
from time import sleep, strftime, time
import argparse
import signal
description = """The fdbcstat utility displays FDB C API statistics on terminal
that include calls-per-second, average latency and maximum latency
within the given time interval.
Each field in the output represents the following elements
in a slash-separated format:
- Operation type
- Number of calls per second
- Average latency in microseconds (us)
- Maximum latency in microseconds (us)
"""
# supported APIs
# note: the array index is important here.
# it's used in BPF as the funciton identifier.
# 0: get
# 1: get_range
# 2: get_read_version
# 3: set
# 4: clear
# 5: clear_range
# 6: commit
fdbfuncs = [
{ "name":"get", "waitfuture":True, "enabled":True },
{ "name":"get_range", "waitfuture":True, "enabled":True },
{ "name":"get_read_version", "waitfuture":True, "enabled":True },
{ "name":"set", "waitfuture":False, "enabled":True },
{ "name":"clear", "waitfuture":False, "enabled":True },
{ "name":"clear_range", "waitfuture":False, "enabled":True },
{ "name":"commit", "waitfuture":True, "enabled":True }
]
# arguments
parser = argparse.ArgumentParser(
description="FoundationDB client statistics collector",
formatter_class=argparse.RawTextHelpFormatter,
epilog=description)
parser.add_argument("-p", "--pid", type=int,
help="Capture for this PID only")
parser.add_argument("-i", "--interval", type=int,
help="Print interval in seconds (Default: 1 second)")
parser.add_argument("-d", "--duration", type=int,
help="Duration in seconds (Default: unset)")
parser.add_argument("-f", "--functions", type=str,
help='''Capture for specific functions (comma-separated) (Default: unset)
Supported functions: get, get_range, get_read_version,
set, clear, clear_range, commit''')
parser.add_argument("libpath",
help="Full path to libfdb_c.so")
args = parser.parse_args()
if not args.interval:
args.interval = 1
if args.functions:
# reset all
idx=0
while idx < len(fdbfuncs):
fdbfuncs[idx]['enabled'] = False
idx += 1
# enable specified functions
for f in args.functions.split(','):
idx=0
while idx < len(fdbfuncs):
if fdbfuncs[idx]['name'] == f:
fdbfuncs[idx]['enabled'] = True
idx += 1
# check for libfdb_c.so
libpath = BPF.find_library(args.libpath) or BPF.find_exe(args.libpath)
if libpath is None:
print("Error: Can't find %s" % args.libpath)
exit(1)
# main BPF program
# we do not rely on PT_REGS_IP() and BPF.sym() to retrive the symbol name
# because some "backword-compatible" symbols do not get resovled through BPF.sym().
bpf_text = """
#include <uapi/linux/ptrace.h>
typedef struct _stats_key_t {
u32 pid;
u32 func;
} stats_key_t;
typedef struct _stats_val_t {
u64 cnt;
u64 total;
u64 max;
} stats_val_t;
BPF_HASH(starttime, u32, u64);
BPF_HASH(startfunc, u32, u32);
BPF_HASH(stats, stats_key_t, stats_val_t);
static int trace_common_entry(struct pt_regs *ctx, u32 func)
{
u64 pid_tgid = bpf_get_current_pid_tgid();
u32 pid = pid_tgid; /* lower 32-bit = Process ID (Thread ID) */
u32 tgid = pid_tgid >> 32; /* upper 32-bit = Thread Group ID (Process ID) */
/* if PID is specified, we'll filter by tgid here */
FILTERPID
/* start time in ns */
u64 ts = bpf_ktime_get_ns();
/* function type */
u32 f = func;
startfunc.update(&pid, &f);
/* update start time */
starttime.update(&pid, &ts);
return 0;
}
int trace_get_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 0);
}
int trace_get_range_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 1);
}
int trace_get_read_version_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 2);
}
int trace_set_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 3);
}
int trace_clear_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 4);
}
int trace_clear_range_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 5);
}
int trace_commit_entry(struct pt_regs *ctx)
{
return trace_common_entry(ctx, 6);
}
int trace_func_return(struct pt_regs *ctx)
{
u64 *st; /* start time */
u64 duration;
u64 pid_tgid = bpf_get_current_pid_tgid();
u32 pid = pid_tgid;
u32 tgid = pid_tgid >> 32;
/* if PID is specified, we'll filter by tgid here */
FILTERPID
/* calculate duration in ns */
st = starttime.lookup(&pid);
if (!st || st == 0) {
return 0; /* missed start */
}
/* duration in ns */
duration = bpf_ktime_get_ns() - *st;
starttime.delete(&pid);
/* update stats */
u32 func, *funcp = startfunc.lookup(&pid);
if (funcp) {
func = *funcp;
stats_key_t key;
stats_val_t *prev;
stats_val_t cur;
key.pid = pid; /* pid here is the thread ID in user space */
key.func = func;
prev = stats.lookup(&key);
if (prev) {
cur.cnt = prev->cnt + 1;
cur.total = prev->total + duration;
cur.max = (duration > prev->max) ? duration : prev->max;
stats.update(&key, &cur);
} else {
cur.cnt = 1;
cur.total = duration;
cur.max = duration;
stats.insert(&key, &cur);
}
startfunc.delete(&pid);
}
return 0;
}
"""
# If PID is specified, insert the PID filter
if args.pid:
bpf_text = bpf_text.replace('FILTERPID',
'if (tgid != %d) { return 0; }' % args.pid)
else:
bpf_text = bpf_text.replace('FILTERPID', '')
# signal handler
def signal_ignore(signal, frame):
pass
# load BPF program
b = BPF(text=bpf_text)
# attach probes
waitfuture = False;
for f in fdbfuncs:
# skip disabled functions
if not f['enabled']:
continue
# attach the entry point
b.attach_uprobe(name=libpath, sym='fdb_transaction_'+f['name'],
fn_name='trace_' + f['name'] + '_entry', pid=args.pid or -1)
if f['waitfuture']:
waitfuture = True
else:
b.attach_uretprobe(name=libpath, sym='fdb_transaction_'+f['name'],
fn_name="trace_func_return", pid=args.pid or -1)
if waitfuture:
b.attach_uretprobe(name=libpath, sym='fdb_future_block_until_ready',
fn_name="trace_func_return", pid=args.pid or -1)
# open uprobes
matched = b.num_open_uprobes()
if matched == 0:
print("0 functions matched... Exiting.")
exit()
stats = b.get_table("stats")
# aggregated stats dictionary
agg = {}
exiting = 0
seconds = 0
prev = 0.0
now = 0.0
# main loop
while (1):
try:
sleep(args.interval)
seconds += args.interval
prev = now
now = time()
if prev == 0:
stats.clear()
continue
except KeyboardInterrupt:
exiting = 1
signal.signal(signal.SIGINT, signal_ignore)
if args.duration and seconds >= args.duration:
exiting = 1
# walk through the stats and aggregate by the functions
for k,v in stats.items():
f = fdbfuncs[k.func]['name']
if f in agg:
# update an exiting entry
agg[f]['cnt'] = agg[f]['cnt'] + v.cnt
agg[f]['total'] = agg[f]['total'] + v.total;
if v.cnt > agg[f]['max']:
agg[f]['max'] = v.cnt
else:
# insert a new entry
agg[f] = {'cnt':v.cnt, 'total':v.total, 'max':v.max}
# print out aggregated stats
print("%-8s " % (strftime("%H:%M:%S")), end="", flush=True)
for f in sorted(agg):
print("%s/%d/%d/%d " % (f,
agg[f]['cnt'] / (now - prev),
agg[f]['total']/agg[f]['cnt'] / 1000, # us
agg[f]['max'] / 1000), # us
end="")
print()
stats.clear()
agg.clear()
if exiting:
exit()

10
contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py
View File

@ -98,6 +98,9 @@ class ByteBuffer(object):
def get_double(self):
return struct.unpack("<d", self.get_bytes(8))[0]
def get_bool(self):
return struct.unpack("<?", self.get_bytes(1))[0]
def get_bytes_with_length(self):
length = self.get_int()
return self.get_bytes(length)
@ -223,6 +226,8 @@ class CommitInfo(BaseInfo):
self.mutations = mutations
self.read_snapshot_version = bb.get_long()
if protocol_version >= PROTOCOL_VERSION_6_3:
self.report_conflicting_keys = bb.get_bool()
class ErrorGetInfo(BaseInfo):
@ -255,7 +260,8 @@ class ErrorCommitInfo(BaseInfo):
self.mutations = mutations
self.read_snapshot_version = bb.get_long()
if protocol_version >= PROTOCOL_VERSION_6_3:
self.report_conflicting_keys = bb.get_bool()
class UnsupportedProtocolVersionError(Exception):
def __init__(self, protocol_version):
@ -546,7 +552,7 @@ class ReadCounter(object):
def get_total_reads(self):
return sum([v for v in self.read_counts.values()])
def matches_filter(addresses, required_addresses):
for addr in required_addresses:
if addr not in addresses:

6
design/special-key-space.md
View File

@ -20,16 +20,16 @@ Consequently, the special-key-space framework wants to integrate all client func
If your feature is exposing information to clients and the results are easily formatted as key-value pairs, then you can use special-key-space to implement your client function.
## How
If you choose to use, you need to implement a function class that inherits from `SpecialKeyRangeBaseImpl`, which has an abstract method `Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr)`.
If you choose to use, you need to implement a function class that inherits from `SpecialKeyRangeReadImpl`, which has an abstract method `Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr)`.
This method can be treated as a callback, whose implementation details are determined by the developer.
Once you fill out the method, register the function class to the corresponding key range.
Below is a detailed example.
```c++
// Implement the function class,
// the corresponding key range is [\xff\xff/example/, \xff\xff/example/\xff)
class SKRExampleImpl : public SpecialKeyRangeBaseImpl {
class SKRExampleImpl : public SpecialKeyRangeReadImpl {
public:
explicit SKRExampleImpl(KeyRangeRef kr): SpecialKeyRangeBaseImpl(kr) {
explicit SKRExampleImpl(KeyRangeRef kr): SpecialKeyRangeReadImpl(kr) {
// Our implementation is quite simple here, the key-value pairs are formatted as:
// \xff\xff/example/<country_name> : <capital_city_name>
CountryToCapitalCity[LiteralStringRef("USA")] = LiteralStringRef("Washington, D.C.");

2
documentation/sphinx/source/administration.rst
View File

@ -228,6 +228,8 @@ If you interrupt the exclude command with Ctrl-C after seeing the "waiting for s
7) If you ever want to add a removed machine back to the cluster, you will have to take it off the excluded servers list to which it was added in step 3. This can be done using the ``include`` command of ``fdbcli``. If attempting to re-include a failed server, this can be done using the ``include failed`` command of ``fdbcli``. Typing ``exclude`` with no parameters will tell you the current list of excluded and failed machines.
As of api version 700, excluding servers can be done with the :ref:`special key space management module <special-key-space-management-module>` as well.
Moving a cluster
================

1
documentation/sphinx/source/api-c.rst
View File

@ -476,6 +476,7 @@ Applications must provide error handling and an appropriate retry loop around th
.. function:: FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length)
Returns an estimated byte size of the key range.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
|future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2|

10
documentation/sphinx/source/api-error-codes.rst
View File

@ -146,6 +146,16 @@ FoundationDB may return the following error codes from API functions. If you nee
| special_keys_no_module_found | 2113| Special key space range read does not intersect a module. |
| | | Refer to the ``SPECIAL_KEY_SPACE_RELAXED`` transaction option for more details.|
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| special_keys_write_disabled | 2114| Special key space is not allowed to write by default. Refer |
| | | to the ``SPECIAL_KEY_SPACE_ENABLE_WRITES`` transaction option for more details.|
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| special_keys_no_write_module_found | 2115| Special key space key or keyrange in set or clear does not intersect a module. |
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| special_keys_cross_module_write | 2116| Special key space clear crosses modules |
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| special_keys_api_failure | 2117| Api call through special keys failed. For more information, read the |
| | | ``0xff0xff/error_message`` key |
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| api_version_unset | 2200| API version is not set |
+-----------------------------------------------+-----+--------------------------------------------------------------------------------+
| api_version_already_set | 2201| API version may be set only once |

1
documentation/sphinx/source/api-python.rst
View File

@ -800,6 +800,7 @@ Transaction misc functions
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. _api-python-transaction-options:

1
documentation/sphinx/source/api-ruby.rst
View File

@ -744,6 +744,7 @@ Transaction misc functions
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
Get the estimated byte size of the given key range. Returns a :class:`Int64Future`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. method:: Transaction.get_approximate_size() -> Int64Future

362
documentation/sphinx/source/architecture.rst
View File

@ -7,3 +7,365 @@ FoundationDB makes your architecture flexible and easy to operate. Your applicat
The following diagram details the logical architecture.
.. image:: /images/Architecture.png
Detailed FoundationDB Architecture
----------------------------------
The FoundationDB architecture chooses a decoupled design, where
processes are assigned different heterogeneous roles (e.g.,
Coordinators, Storage Servers, Master). Scaling the database is achieved
by horizontally expanding the number of processes for separate roles:
Coordinators
~~~~~~~~~~~~
All clients and servers connect to a FoundationDB cluster with a cluster
file, which contains the IP:PORT of the coordinators. Both the clients
and servers use the coordinators to connect with the cluster controller.
The servers will attempt to become the cluster controller if one does
not exist, and register with the cluster controller once one has been
elected. Clients use the cluster controller to keep an up-to-date list
of proxies.
Cluster Controller
~~~~~~~~~~~~~~~~~~
The cluster controller is a singleton elected by a majority of
coordinators. It is the entry point for all processes in the cluster. It
is responsible for determining when a process has failed, telling
processes which roles they should become, and passing system information
between all of the processes.
Master
~~~~~~
The master is responsible for coordinating the transition of the write
sub-system from one generation to the next. The write sub-system
includes the master, proxies, resolvers, and transaction logs. The three
roles are treated as a unit, and if any of them fail, we will recruit a
replacement for all three roles. The master provides the commit versions
for batches of the mutations to the proxies.
Historically, Ratekeeper and Data Distributor are coupled with Master on
the same process. Since 6.2, both have become a singleton in the
cluster. The life time is no longer tied with Master.
|image1|
Proxies
~~~~~~~
The proxies are responsible for providing read versions, committing
transactions, and tracking the storage servers responsible for each
range of keys. To provide a read version, a proxy will ask all other
proxies to see the largest committed version at this point in time,
while simultaneously checking that the transaction logs have not been
stopped. Ratekeeper will artificially slow down the rate at which the
proxy provides read versions.
Commits are accomplished by:
- Get a commit version from the master.
- Use the resolvers to determine if the transaction conflicts with
previously committed transactions.
- Make the transaction durable on the transaction logs.
The key space starting with the ``\xff`` byte is reserved for system
metadata. All mutations committed into this key space are distributed to
all of the proxies through the resolvers. This metadata includes a
mapping between key ranges and the storage servers which have the data
for that range of keys. The proxies provides this information to clients
on-demand. The clients cache this mapping; if they ask a storage server
for a key it does not have, they will clear their cache and get a more
up-to-date list of servers from the proxies.
Transaction Logs
~~~~~~~~~~~~~~~~
The transaction logs make mutations durable to disk for fast commit
latencies. The logs receive commits from the proxy in version order, and
only respond to the proxy once the data has been written and fsync’ed to
an append only mutation log on disk. Before the data is even written to
disk we forward it to the storage servers responsible for that mutation.
Once the storage servers have made the mutation durable, they pop it
from the log. This generally happens roughly 6 seconds after the
mutation was originally committed to the log. We only read from the
log’s disk when the process has been rebooted. If a storage server has
failed, mutations bound for that storage server will build up on the
logs. Once data distribution makes a different storage server
responsible for all of the missing storage server’s data we will discard
the log data bound for the failed server.
Resolvers
~~~~~~~~~
The resolvers are responsible determining conflicts between
transactions. A transaction conflicts if it reads a key that has been
written between the transaction’s read version and commit version. The
resolver does this by holding the last 5 seconds of committed writes in
memory, and comparing a new transaction’s reads against this set of
commits.
Storage Servers
~~~~~~~~~~~~~~~
The vast majority of processes in a cluster are storage servers. Storage
servers are assigned ranges of key, and are responsible to storing all
of the data for that range. They keep 5 seconds of mutations in memory,
and an on disk copy of the data as of 5 second ago. Clients must read at
a version within the last 5 seconds, or they will get a
``transaction_too_old`` error. The SSD storage engine stores the data in
a B-tree based on SQLite. The memory storage engine store the data in
memory with an append only log that is only read from disk if the
process is rebooted. In the upcoming FoundationDB 7.0 release, the
B-tree storage engine will be replaced with a brand new *Redwood*
engine.
Data Distributor
~~~~~~~~~~~~~~~~
Data distributor manages the lifetime of storage servers, decides which
storage server is responsible for which data range, and ensures data is
evenly distributed across all storage servers (SS). Data distributor as
a singleton in the cluster is recruited and monitored by Cluster
Controller. See `internal
documentation <https://github.com/apple/foundationdb/blob/master/design/data-distributor-internals.md>`__.
Ratekeeper
~~~~~~~~~~
Ratekeeper monitors system load and slows down client transaction rate
when the cluster is close to saturation by lowering the rate at which
the proxy provides read versions. Ratekeeper as a singleton in the
cluster is recruited and monitored by Cluster Controller.
Clients
~~~~~~~
A client links with specific language bindings (i.e., client libraries)
in order to communicate with a FoundationDB cluster. The language
bindings support loading multiple versions of C libraries, allowing the
client communicates with older version of the FoundationDB clusters.
Currently, C, Go, Python, Java, Ruby bindings are officially supported.
Transaction Processing
----------------------
A database transaction in FoundationDB starts by a client contacting one
of the Proxies to obtain a read version, which is guaranteed to be
larger than any of commit version that client may know about (even
through side channels outside the FoundationDB cluster). This is needed
so that a client will see the result of previous commits that have
happened.
Then the client may issue multiple reads to storage servers and obtain
values at that specific read version. Client writes are kept in local
memory without contacting the cluster. By default, reading a key that
was written in the same transaction will return the newly written value.
At commit time, the client sends the transaction data (all reads and
writes) to one of the Proxies and waits for commit or abort response
from the proxy. If the transaction conflicts with another one and cannot
commit, the client may choose to retry the transaction from the
beginning again. If the transaction commits, the proxy also returns the
commit version back to the client. Note this commit version is larger
than the read version and is chosen by the master.
The FoundationDB architecture separates the scaling of client reads and
writes (i.e., transaction commits). Because clients directly issue reads
to sharded storage servers, reads scale linearly to the number of
storage servers. Similarly, writes are scaled by adding more processes
to Proxies, Resolvers, and Log Servers in the transaction system.
Determine Read Version
~~~~~~~~~~~~~~~~~~~~~~
When a client requests a read version from a proxy, the proxy asks all
other proxies for their last commit versions, and checks a set of
transaction logs satisfying replication policy are live. Then the proxy
returns the maximum commit version as the read version to the client.
|image2|
The reason for the proxy to contact all other proxies for commit
versions is to ensure the read version is larger than any previously
committed version. Consider that if proxy ``A`` commits a transaction,
and then the client asks proxy ``B`` for a read version. The read
version from proxy ``B`` must be larger than the version committed by
proxy ``A``. The only way to get this information is by asking proxy
``A`` for its largest committed version.
The reason for checking a set of transaction logs satisfying replication
policy are live is to ensure the proxy is not replaced with newer
generation of proxies. This is because proxy is a stateless role
recruited in each generation. If a recovery has happened and the old
proxy is still live, this old proxy could still give out read versions.
As a result, a *read-only* transaction may see stale results (a
read-write transaction will be aborted). By checking a set of
transaction logs satisfying replication policy are live, the proxy makes
sure no recovery has happened, thus the *read-only* transaction sees the
latest data.
Note that the client cannot simply ask the master for read versions. The
master gives out versions to proxies to be committed, but the master
does not know when the versions it gives out are durable on the
transaction logs. Therefore it is not safe to do reads at the largest
version the master has provided because that version might be rolled
back in the event of a failure, so the client could end up reading data
that was never committed. In order for the client to use versions from
the master, the client needs to wait until all in-flight
transaction-batches (a write version is used for a batch of
transactions) to commit. This can take a long time and thus is
inefficient. Another drawback of this approach is putting more work
towards the master, because the master role can’t be scaled. Even though
giving out read-versions isn’t very expensive, it still requires the
master to get a transaction budget from the Ratekeeper, batches
requests, and potentially maintains thousands of network connections
from clients.
|image3|
Transaction Commit
~~~~~~~~~~~~~~~~~~
A client transaction commits in the following steps:
1. A client sends a transaction to a proxy.
2. The proxy asks the master for a commit version.
3. The master sends back a commit version that is higher than any commit
version seen before.
4. The proxy sends the read and write conflict ranges to the resolver(s)
with the commit version included.
5. The resolver responds back with whether the transaction has any
conflicts with previous transactions by sorting transactions
according to their commit versions and computing if such a serial
execution order is conflict-free.
- If there are conflicts, the proxy responds back to the client with
a not_committed error.
- If there are no conflicts, the proxy sends the mutations and
commit version of this transaction to the transaction logs.
6. Once the mutations are durable on the logs, the proxy responds back
success to the user.
Note the proxy sends each resolver their respective key ranges, if any
one of the resolvers detects a conflict then the transaction is not
committed. This has the flaw that if only one of the resolvers detects a
conflict, the other resolver will still think the transaction has
succeeded and may fail future transactions with overlapping write
conflict ranges, even though these future transaction can commit. In
practice, a well designed workload will only have a very small
percentage of conflicts, so this amplification will not affect
performance. Additionally, each transaction has a five seconds window.
After five seconds, resolvers will remove the conflict ranges of old
transactions, which also limits the chance of this type of false
conflict.
|image4|
|image5|
Background Work
~~~~~~~~~~~~~~~
There are a number of background work happening besides the transaction
processing:
- **Ratekeeper** collects statistic information from proxies,
transaction logs, and storage servers and compute the target
transaction rate for the cluster.
- **Data distribution** monitors all storage servers and perform load
balancing operations to evenly distribute data among all storage
servers.
- **Storage servers** pull mutations from transaction logs, write them
into storage engine to persist on disks.
- **Proxies** periodically send empty commits to transaction logs to
keep commit versions increasing, in case there is no client generated
transactions.
|image6|
Transaction System Recovery
~~~~~~~~~~~~~~~~~~~~~~~~~~~
The transaction system implements the write pipeline of the FoundationDB
cluster and its performance is critical to the transaction commit
latency. A typical recovery takes about a few hundred milliseconds, but
longer recovery time (usually a few seconds) can happen. Whenever there
is a failure in the transaction system, a recovery process is performed
to restore the transaction system to a new configuration, i.e., a clean
state. Specifically, the Master process monitors the health of Proxies,
Resolvers, and Transaction Logs. If any one of the monitored process
failed, the Master process terminates. The Cluster Controller will
detect this event, and then recruits a new Master, which coordinates the
recovery and recruits a new transaction system instance. In this way,
the transaction processing is divided into a number of epochs, where
each epoch represents a generation of the transaction system with its
unique Master process.
For each epoch, the Master initiates recovery in several steps. First,
the Master reads the previous transaction system states from
Coordinators and lock the coordinated states to prevent another Master
process from recovering at the same time. Then the Master recovers
previous transaction system states, including all Log Servers’
Information, stops these Log Servers from accepting transactions, and
recruits a new set of Proxies, Resolvers, and Transaction Logs. After
previous Log Servers are stopped and new transaction system is
recruited, the Master writes the coordinated states with current
transaction system information. Finally, the Master accepts new
transaction commits. See details in this
`documentation <https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md>`__.
Because Proxies and Resolvers are stateless, their recoveries have no
extra work. In contrast, Transaction Logs save the logs of committed
transactions, and we need to ensure all previously committed
transactions are durable and retrievable by storage servers. That is,
for any transactions that the Proxies may have sent back commit
response, their logs are persisted in multiple Log Servers (e.g., three
servers if replication degree is 3).
Finally, a recovery will *fast forward* time by 90 seconds, which would
abort any in-progress client transactions with ``transaction_too_old``
error. During retry, these client transactions will find the new
generation of transaction system and commit.
**``commit_result_unknown`` error:** If a recovery happened while a
transaction is committing (i.e., a proxy has sent mutations to
transaction logs). A client would have received
``commit_result_unknown``, and then retried the transaction. It’s
completely permissible for FDB to commit both the first attempt, and the
second retry, as ``commit_result_unknown`` means the transaction may or
may not have committed. This is why it’s strongly recommended that
transactions should be idempotent, so that they handle
``commit_result_unknown`` correctly.
Resources
---------
`Forum
Post <https://forums.foundationdb.org/t/technical-overview-of-the-database/135/26>`__
`Existing Architecture
Documentation <https://github.com/apple/foundationdb/blob/master/documentation/sphinx/source/kv-architecture.rst>`__
`Summit
Presentation <https://www.youtube.com/watch?list=PLbzoR-pLrL6q7uYN-94-p_-Q3hyAmpI7o&v=EMwhsGsxfPU&feature=emb_logo>`__
`Data Distribution
Documentation <https://github.com/apple/foundationdb/blob/master/design/data-distributor-internals.md>`__
`Recovery
Documentation <https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md>`__
.. |image1| image:: images/architecture-1.jpeg
.. |image2| image:: images/architecture-2.jpeg
.. |image3| image:: images/architecture-3.jpeg
.. |image4| image:: images/architecture-4.jpeg
.. |image5| image:: images/architecture-5.jpeg
.. |image6| image:: images/architecture-6.jpeg

2
documentation/sphinx/source/configuration.rst
View File

@ -583,7 +583,7 @@ Clients should also specify their datacenter with the database option ``datacent
Changing the region configuration
---------------------------------
To change the region configure, use the ``fileconfigure`` command ``fdbcli``. For example::
To change the region configuration, use the ``fileconfigure`` command ``fdbcli``. For example::
user@host$ fdbcli
Using cluster file `/etc/foundationdb/fdb.cluster'.

58
documentation/sphinx/source/developer-guide.rst
View File

@ -763,8 +763,8 @@ Special keys
Keys starting with the bytes ``\xff\xff`` are called "special" keys, and they are materialized when read. :doc:`\\xff\\xff/status/json <mr-status>` is an example of a special key.
As of api version 630, additional features have been exposed as special keys and are available to read as ranges instead of just individual keys. Additionally, the special keys are now organized into "modules".
Modules
-------
Read-only modules
-----------------
A module is loosely defined as a key range in the special key space where a user can expect similar behavior from reading any key in that range.
By default, users will see a ``special_keys_no_module_found`` error if they read from a range not contained in a module.
@ -912,6 +912,59 @@ Caveats
#. ``\xff\xff/metrics/health/`` These keys may return data that's several seconds old, and the data may not be available for a brief period during recovery. This will be indicated by the keys being absent.
Read/write modules
------------------
As of api version 700, some modules in the special key space allow writes as
well as reads. In these modules, a user can expect that mutations (i.e. sets,
clears, etc) do not have side-effects outside of the current transaction
until commit is called (the same is true for writes to the normal key space).
A user can also expect the effects on commit to be atomic. Reads to
special keys may require reading system keys (whose format is an implementation
detail), and for those reads appropriate read conflict ranges are added on
the underlying system keys.
Writes to read/write modules in the special key space are disabled by
default. Use the ``special_key_space_enable_writes`` transaction option to
enable them [#special_key_space_enable_writes]_.
.. _special-key-space-management-module:
Management module
~~~~~~~~~~~~~~~~~
The management module is for temporary cluster configuration changes. For
example, in order to safely remove a process from the cluster, one can add an
exclusion to the ``\xff\xff/management/excluded/`` key prefix that matches
that process, and wait for necessary data to be moved away.
#. ``\xff\xff/management/excluded/<exclusion>`` Read/write. Indicates that the cluster should move data away from processes matching ``<exclusion>``, so that they can be safely removed. See :ref:`removing machines from a cluster <removing-machines-from-a-cluster>` for documentation for the corresponding fdbcli command.
#. ``\xff\xff/management/failed/<exclusion>`` Read/write. Indicates that the cluster should consider matching processes as permanently failed. This allows the cluster to avoid maintaining extra state and doing extra work in the hope that these processes come back. See :ref:`removing machines from a cluster <removing-machines-from-a-cluster>` for documentation for the corresponding fdbcli command.
#. ``\xff\xff/management/inProgressExclusion/<address>`` Read-only. Indicates that the process matching ``<address>`` matches an exclusion, but still has necessary data and can't yet be safely removed.
#. ``\xff\xff/management/options/excluded/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/excluded/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
#. ``\xff\xff/management/options/failed/force`` Read/write. Setting this key disables safety checks for writes to ``\xff\xff/management/failed/<exclusion>``. Setting this key only has an effect in the current transaction and is not persisted on commit.
An exclusion is syntactically either an ip address (e.g. ``127.0.0.1``), or
an ip address and port (e.g. ``127.0.0.1:4500``). If no port is specified,
then all processes on that host match the exclusion.
Error message module
~~~~~~~~~~~~~~~~~~~~
Each module written to validates the transaction before committing, and this
validation failing is indicated by a ``special_keys_api_failure`` error.
More detailed information about why this validation failed can be accessed through the ``\xff\xff/error_message`` key, whose value is a json document with the following schema.
========================== ======== ===============
**Field** **Type** **Description**
-------------------------- -------- ---------------
retriable boolean Whether or not this operation might succeed if retried
command string The fdbcli command corresponding to this operation
message string Help text explaining the reason this operation failed
========================== ======== ===============
Performance considerations
==========================
@ -1114,3 +1167,4 @@ At a first glance this looks very similar to an ``commit_unknown_result``. Howev
.. [#conflicting_keys] In practice, the transaction probably committed successfully. However, if you're running multiple resolvers then it's possible for a transaction to cause another to abort even if it doesn't commit successfully.
.. [#max_read_transaction_life_versions] The number 5000000 comes from the server knob MAX_READ_TRANSACTION_LIFE_VERSIONS
.. [#special_key_space_enable_writes] Enabling this option enables other transaction options, such as ``ACCESS_SYSTEM_KEYS``. This may change in the future.

24
documentation/sphinx/source/downloads.rst
View File

@ -10,38 +10,38 @@ macOS
The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
* `FoundationDB-6.3.3.pkg <https://www.foundationdb.org/downloads/6.3.3/macOS/installers/FoundationDB-6.3.3.pkg>`_
* `FoundationDB-6.3.4.pkg <https://www.foundationdb.org/downloads/6.3.4/macOS/installers/FoundationDB-6.3.4.pkg>`_
Ubuntu
------
The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
* `foundationdb-clients-6.3.3-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.3/ubuntu/installers/foundationdb-clients_6.3.3-1_amd64.deb>`_
* `foundationdb-server-6.3.3-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.3/ubuntu/installers/foundationdb-server_6.3.3-1_amd64.deb>`_ (depends on the clients package)
* `foundationdb-clients-6.3.4-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.4/ubuntu/installers/foundationdb-clients_6.3.4-1_amd64.deb>`_
* `foundationdb-server-6.3.4-1_amd64.deb <https://www.foundationdb.org/downloads/6.3.4/ubuntu/installers/foundationdb-server_6.3.4-1_amd64.deb>`_ (depends on the clients package)
RHEL/CentOS EL6
---------------
The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
* `foundationdb-clients-6.3.3-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.3/rhel6/installers/foundationdb-clients-6.3.3-1.el6.x86_64.rpm>`_
* `foundationdb-server-6.3.3-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.3/rhel6/installers/foundationdb-server-6.3.3-1.el6.x86_64.rpm>`_ (depends on the clients package)
* `foundationdb-clients-6.3.4-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.4/rhel6/installers/foundationdb-clients-6.3.4-1.el6.x86_64.rpm>`_
* `foundationdb-server-6.3.4-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.4/rhel6/installers/foundationdb-server-6.3.4-1.el6.x86_64.rpm>`_ (depends on the clients package)
RHEL/CentOS EL7
---------------
The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
* `foundationdb-clients-6.3.3-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.3/rhel7/installers/foundationdb-clients-6.3.3-1.el7.x86_64.rpm>`_
* `foundationdb-server-6.3.3-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.3/rhel7/installers/foundationdb-server-6.3.3-1.el7.x86_64.rpm>`_ (depends on the clients package)
* `foundationdb-clients-6.3.4-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.4/rhel7/installers/foundationdb-clients-6.3.4-1.el7.x86_64.rpm>`_
* `foundationdb-server-6.3.4-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.3.4/rhel7/installers/foundationdb-server-6.3.4-1.el7.x86_64.rpm>`_ (depends on the clients package)
Windows
-------
The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
* `foundationdb-6.3.3-x64.msi <https://www.foundationdb.org/downloads/6.3.3/windows/installers/foundationdb-6.3.3-x64.msi>`_
* `foundationdb-6.3.4-x64.msi <https://www.foundationdb.org/downloads/6.3.4/windows/installers/foundationdb-6.3.4-x64.msi>`_
API Language Bindings
=====================
@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package:
* `foundationdb-6.3.3.tar.gz <https://www.foundationdb.org/downloads/6.3.3/bindings/python/foundationdb-6.3.3.tar.gz>`_
* `foundationdb-6.3.4.tar.gz <https://www.foundationdb.org/downloads/6.3.4/bindings/python/foundationdb-6.3.4.tar.gz>`_
Ruby 1.9.3/2.0.0+
-----------------
* `fdb-6.3.3.gem <https://www.foundationdb.org/downloads/6.3.3/bindings/ruby/fdb-6.3.3.gem>`_
* `fdb-6.3.4.gem <https://www.foundationdb.org/downloads/6.3.4/bindings/ruby/fdb-6.3.4.gem>`_
Java 8+
-------
* `fdb-java-6.3.3.jar <https://www.foundationdb.org/downloads/6.3.3/bindings/java/fdb-java-6.3.3.jar>`_
* `fdb-java-6.3.3-javadoc.jar <https://www.foundationdb.org/downloads/6.3.3/bindings/java/fdb-java-6.3.3-javadoc.jar>`_
* `fdb-java-6.3.4.jar <https://www.foundationdb.org/downloads/6.3.4/bindings/java/fdb-java-6.3.4.jar>`_
* `fdb-java-6.3.4-javadoc.jar <https://www.foundationdb.org/downloads/6.3.4/bindings/java/fdb-java-6.3.4-javadoc.jar>`_
Go 1.11+
--------

BIN
documentation/sphinx/source/images/architecture-1.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 142 KiB

BIN
documentation/sphinx/source/images/architecture-2.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 139 KiB

BIN
documentation/sphinx/source/images/architecture-3.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 152 KiB

BIN
documentation/sphinx/source/images/architecture-4.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 143 KiB

BIN
documentation/sphinx/source/images/architecture-5.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 160 KiB

BIN
documentation/sphinx/source/images/architecture-6.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 159 KiB

8
documentation/sphinx/source/release-notes/release-notes-620.rst
View File

@ -2,6 +2,14 @@
Release Notes
#############
6.2.24
======
Features
--------
* Added the ``suspend`` command to ``fdbcli`` which kills a process and prevents it from rejoining the cluster for a specified duration. `(PR #3550) <https://github.com/apple/foundationdb/pull/3550>`_
6.2.23
======

11
documentation/sphinx/source/release-notes/release-notes-630.rst
View File

@ -2,7 +2,13 @@
Release Notes
#############
6.3.3
6.3.5
=====
* Fix an issue where ``fdbcli --exec 'exclude no_wait ...'`` would incorrectly report that processes can safely be removed from the cluster. `(PR #3566) <https://github.com/apple/foundationdb/pull/3566>`_
* When a configuration key is changed, it will always be included in ``status json`` output, even the value is reverted back to the default value. `(PR #3610) <https://github.com/apple/foundationdb/pull/3610>`_
6.3.4
=====
Features
@ -36,6 +42,7 @@ Performance
* Reduced the number of comparisons used by various map implementations. `(PR #2882) <https://github.com/apple/foundationdb/pull/2882>`_
* Reduced the serialized size of empty strings. `(PR #3063) <https://github.com/apple/foundationdb/pull/3063>`_
* Reduced the serialized size of various interfaces by 10x. `(PR #3068) <https://github.com/apple/foundationdb/pull/3068>`_
* TLS handshakes can now be done in a background thread pool. `(PR #3403) <https://github.com/apple/foundationdb/pull/3403>`_
Reliability
-----------
@ -57,6 +64,8 @@ Fixes
* Transaction logs configured to spill by reference had an unintended delay between each spilled batch. `(PR #3153) <https://github.com/apple/foundationdb/pull/3153>`_
* Added guards to honor ``DISABLE_POSIX_KERNEL_AIO``. `(PR #2888) <https://github.com/apple/foundationdb/pull/2888>`_
* Prevent blob upload timeout if request timeout is lower than expected request time. `(PR #3533) <https://github.com/apple/foundationdb/pull/3533>`_
* In very rare scenarios, the data distributor process would crash when being shutdown. `(PR #3530) <https://github.com/apple/foundationdb/pull/3530>`_
* The master would die immediately if it did not have the correct cluster controller interface when recruited. [6.3.4] `(PR #3537) <https://github.com/apple/foundationdb/pull/3537>`_
Status
------

2
fdbbackup/FileConverter.actor.cpp
View File

@ -233,7 +233,7 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
void dumpProgress(std::string msg) {
std::cout << msg << "\n ";
for (const auto fp : fileProgress) {
for (const auto& fp : fileProgress) {
std::cout << fp->fd->getFilename() << " " << fp->mutations.size() << " mutations";
if (fp->mutations.size() > 0) {
std::cout << ", range " << fp->mutations[0].version.toString() << " "

97
fdbbackup/backup.actor.cpp
View File

@ -623,6 +623,7 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
SO_END_OF_OPTIONS
};
// g_rgRestoreOptions is used by fdbrestore and fastrestore_tool
CSimpleOpt::SOption g_rgRestoreOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
@ -1064,9 +1065,9 @@ static void printRestoreUsage(bool devhelp ) {
printf(" Prefix to add to the restored keys\n");
printf(" -n, --dryrun Perform a trial run with no changes made.\n");
printf(" --log Enables trace file logging for the CLI session.\n"
" --logdir PATH Specifes the output directory for trace files. If\n"
" unspecified, defaults to the current directory. Has\n"
" no effect unless --log is specified.\n");
" --logdir PATH Specifies the output directory for trace files. If\n"
" unspecified, defaults to the current directory. Has\n"
" no effect unless --log is specified.\n");
printf(" --loggroup LOG_GROUP\n"
" Sets the LogGroup field with the specified value for all\n"
" events in the trace output (defaults to `default').\n");
@ -1104,43 +1105,10 @@ static void printRestoreUsage(bool devhelp ) {
}
static void printFastRestoreUsage(bool devhelp) {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("Usage: %s (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str());
// printf(" FOLDERS Paths to folders containing the backup files.\n");
printf("Options for all commands:\n\n");
printf(" -C CONNFILE The path of a file containing the connection string for the\n"
" FoundationDB cluster. The default is first the value of the\n"
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
" then `%s'.\n",
platform::getDefaultClusterFilePath().c_str());
printf(" -t TAGNAME The restore tag to act on. Default is 'default'\n");
printf(" --tagname TAGNAME\n\n");
printf(" Options for start:\n\n");
printf(" -r URL The Backup URL for the restore to read from.\n");
printBackupContainerInfo();
printf(" -w Wait for the restore to complete before exiting. Prints progress updates.\n");
printf(" --waitfordone\n");
printf(" -k KEYS List of key ranges from the backup to restore\n");
printf(" --remove_prefix PREFIX prefix to remove from the restored keys\n");
printf(" --add_prefix PREFIX prefix to add to the restored keys\n");
printf(" -n, --dry-run Perform a trial run with no changes made.\n");
printf(" -v DBVERSION The version at which the database will be restored.\n");
printf(" -h, --help Display this help and exit.\n");
printf("NOTE: Fast restore is still under development. The options may not be fully supported.\n");
if (devhelp) {
#ifdef _WIN32
printf(" -q Disable error dialog on crash.\n");
printf(" --parentpid PID\n");
printf(" Specify a process after whose termination to exit.\n");
#endif
}
printf("\n"
" KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
printf("\n");
puts(BlobCredentialInfo);
printf(" NOTE: Fast restore aims to support the same fdbrestore option list.\n");
printf(" But fast restore is still under development. The options may not be fully supported.\n");
printf(" Supported options are: --dest_cluster_file, -r, --waitfordone, --logdir\n");
printRestoreUsage(devhelp);
return;
}
@ -2260,6 +2228,11 @@ ACTOR Future<Void> runFastRestoreTool(Database db, std::string tagName, std::str
printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n", ranges.size(),
ranges.front().toString().c_str());
TraceEvent ev("FastRestoreTool");
ev.detail("RestoreRanges", ranges.size());
for (int i = 0; i < ranges.size(); ++i) {
ev.detail(format("Range%d", i), ranges[i]);
}
if (performRestore) {
if (dbVersion == invalidVersion) {
@ -3390,7 +3363,7 @@ int main(int argc, char* argv[]) {
break;
case EXE_FASTRESTORE_TOOL:
fprintf(stderr, "ERROR: FDB Fast Restore Agent does not support argument value `%s'\n",
fprintf(stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n",
args->File(argLoop));
printHelpTeaser(argv[0]);
return FDB_EXIT_ERROR;
@ -3807,12 +3780,40 @@ int main(int argc, char* argv[]) {
}
break;
case EXE_FASTRESTORE_TOOL:
// TODO: We have not implmented the code commented out in this case
if (!initCluster()) return FDB_EXIT_ERROR;
// Support --dest_cluster_file option as fdbrestore does
if (dryRun) {
if (restoreType != RESTORE_START) {
fprintf(stderr, "Restore dry run only works for 'start' command\n");
return FDB_EXIT_ERROR;
}
// Must explicitly call trace file options handling if not calling Database::createDatabase()
initTraceFile();
} else {
if (restoreClusterFileDest.empty()) {
fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
return FDB_EXIT_ERROR;
}
if (!fileExists(restoreClusterFileDest)) {
fprintf(stderr, "Restore destination cluster file '%s' does not exist.\n",
restoreClusterFileDest.c_str());
return FDB_EXIT_ERROR;
}
try {
db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST);
} catch (Error& e) {
fprintf(stderr, "Restore destination cluster file '%s' invalid: %s\n",
restoreClusterFileDest.c_str(), e.what());
return FDB_EXIT_ERROR;
}
}
// TODO: We have not implemented the code commented out in this case
switch (restoreType) {
case RESTORE_START:
f = stopAfter(runFastRestoreTool(db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun,
!quietDisplay, waitForDone));
!quietDisplay, waitForDone));
break;
case RESTORE_WAIT:
printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n");
@ -3823,8 +3824,9 @@ int main(int argc, char* argv[]) {
printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n");
throw restore_error();
// f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)),
//[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State: %s\n", tagName.c_str(),
//FileBackupAgent::restoreStateText(s).toString().c_str()); return Void();
//[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State: %s\n",
//tagName.c_str(),
// FileBackupAgent::restoreStateText(s).toString().c_str()); return Void();
// }) );
break;
case RESTORE_STATUS:
@ -3904,7 +3906,8 @@ int main(int argc, char* argv[]) {
<< FastAllocator<1024>::pageCount << " "
<< FastAllocator<2048>::pageCount << " "
<< FastAllocator<4096>::pageCount << " "
<< FastAllocator<8192>::pageCount << endl;
<< FastAllocator<8192>::pageCount << " "
<< FastAllocator<16384>::pageCount << endl;
vector< std::pair<std::string, const char*> > typeNames;
for( auto i = allocInstr.begin(); i != allocInstr.end(); ++i ) {

197
fdbcli/fdbcli.actor.cpp
View File

@ -556,6 +556,10 @@ void initHelp() {
"kill all|list|<ADDRESS...>",
"attempts to kill one or more processes in the cluster",
"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is unresponsive.\n\nFor each IP:port pair in <ADDRESS ...>, attempt to kill the specified process.");
helpMap["suspend"] = CommandHelp(
"suspend <SECONDS> <ADDRESS...>",
"attempts to suspend one or more processes in the cluster",
"If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be suspended before this list has been populated.\n\nFor each IP:port pair in <ADDRESS...>, attempt to suspend the processes for the specified SECONDS after which the process will die.");
helpMap["profile"] = CommandHelp(
"profile <client|list|flow|heap> <action> <ARGS>",
"namespace for all the profiling-related commands.",
@ -577,6 +581,11 @@ void initHelp() {
"view and control throttled tags",
"Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n"
);
helpMap["cache_range"] = CommandHelp(
"cache_range <set|clear> <BEGINKEY> <ENDKEY>",
"Mark a key range to add to or remove from storage caches.",
"Use the storage caches to assist in balancing hot read shards. Set the appropriate ranges when experiencing heavy load, and clear them when they are no longer necessary."
);
helpMap["lock"] = CommandHelp(
"lock",
"lock the database with a randomly generated lockUID",
@ -2150,8 +2159,8 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
return false;
} else {
state std::vector<AddressExclusion> addresses;
state std::set<AddressExclusion> exclusions;
state std::vector<AddressExclusion> exclusionVector;
state std::set<AddressExclusion> exclusionSet;
bool force = false;
state bool waitForAllExcluded = true;
state bool markFailed = false;
@ -2170,8 +2179,8 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
printf(" Do not include the `:tls' suffix when naming a process\n");
return true;
}
addresses.push_back( a );
exclusions.insert( a );
exclusionVector.push_back(a);
exclusionSet.insert(a);
}
}
@ -2179,9 +2188,10 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
if (markFailed) {
state bool safe;
try {
bool _safe = wait(makeInterruptable(checkSafeExclusions(db, addresses)));
bool _safe = wait(makeInterruptable(checkSafeExclusions(db, exclusionVector)));
safe = _safe;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
TraceEvent("CheckSafeExclusionsError").error(e);
safe = false;
}
@ -2240,7 +2250,8 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
return true;
}
NetworkAddress addr = NetworkAddress::parse(addrStr);
bool excluded = (process.has("excluded") && process.last().get_bool()) || addressExcluded(exclusions, addr);
bool excluded =
(process.has("excluded") && process.last().get_bool()) || addressExcluded(exclusionSet, addr);
ssTotalCount++;
if (excluded)
ssExcludedCount++;
@ -2281,7 +2292,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
}
}
wait(makeInterruptable(excludeServers(db, addresses, markFailed)));
wait(makeInterruptable(excludeServers(db, exclusionVector, markFailed)));
if (waitForAllExcluded) {
printf("Waiting for state to be removed from all excluded servers. This may take a while.\n");
@ -2292,7 +2303,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
warn.cancel();
state std::set<NetworkAddress> notExcludedServers =
wait(makeInterruptable(checkForExcludingServers(db, addresses, waitForAllExcluded)));
wait(makeInterruptable(checkForExcludingServers(db, exclusionVector, waitForAllExcluded)));
std::vector<ProcessData> workers = wait( makeInterruptable(getWorkers(db)) );
std::map<IPAddress, std::set<uint16_t>> workerPorts;
for(auto addr : workers)
@ -2300,7 +2311,7 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
// Print a list of all excluded addresses that don't have a corresponding worker
std::set<AddressExclusion> absentExclusions;
for(auto addr : addresses) {
for (const auto& addr : exclusionVector) {
auto worker = workerPorts.find(addr.ip);
if(worker == workerPorts.end())
absentExclusions.insert(addr);
@ -2308,43 +2319,46 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
absentExclusions.insert(addr);
}
for (auto addr : addresses) {
NetworkAddress _addr(addr.ip, addr.port);
if (absentExclusions.find(addr) != absentExclusions.end()) {
if(addr.port == 0)
for (const auto& exclusion : exclusionVector) {
if (absentExclusions.find(exclusion) != absentExclusions.end()) {
if (exclusion.port == 0) {
printf(" %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the "
"correct machines before removing them from the cluster!\n",
addr.ip.toString().c_str());
else
exclusion.ip.toString().c_str());
} else {
printf(" %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes "
"before removing them from the cluster!\n",
addr.toString().c_str());
} else if (notExcludedServers.find(_addr) != notExcludedServers.end()) {
if (addr.port == 0)
exclusion.toString().c_str());
}
} else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(),
[&](const NetworkAddress& a) { return addressExcluded({ exclusion }, a); })) {
if (exclusion.port == 0) {
printf(" %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this "
"machine from the cluster\n",
addr.ip.toString().c_str());
else
exclusion.ip.toString().c_str());
} else {
printf(" %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the "
"cluster\n",
addr.toString().c_str());
exclusion.toString().c_str());
}
} else {
if (addr.port == 0)
if (exclusion.port == 0) {
printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine "
"from the cluster.\n",
addr.ip.toString().c_str());
else
exclusion.ip.toString().c_str());
} else {
printf(
" %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n",
addr.toString().c_str());
exclusion.toString().c_str());
}
}
}
bool foundCoordinator = false;
auto ccs = ClusterConnectionFile( ccf->getFilename() ).getConnectionString();
for( auto& c : ccs.coordinators()) {
if (std::count( addresses.begin(), addresses.end(), AddressExclusion(c.ip, c.port) ) ||
std::count( addresses.begin(), addresses.end(), AddressExclusion(c.ip) )) {
if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
printf("WARNING: %s is a coordinator!\n", c.toString().c_str());
foundCoordinator = true;
}
@ -3365,7 +3379,11 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
for( auto it : address_interface ) {
tr->set(LiteralStringRef("\xff\xff/reboot_worker"), it.second.first);
if (db->apiVersionAtLeast(700))
BinaryReader::fromStringRef<ClientWorkerInterface>(it.second.first, IncludeVersion())
.reboot.send(RebootRequest());
else
tr->set(LiteralStringRef("\xff\xff/reboot_worker"), it.second.first);
}
if (address_interface.size() == 0) {
printf("ERROR: no processes to kill. You must run the `kill’ command before running `kill all’.\n");
@ -3383,7 +3401,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if(!is_error) {
for(int i = 1; i < tokens.size(); i++) {
tr->set(LiteralStringRef("\xff\xff/reboot_worker"), address_interface[tokens[i]].first);
if (db->apiVersionAtLeast(700))
BinaryReader::fromStringRef<ClientWorkerInterface>(
address_interface[tokens[i]].first, IncludeVersion())
.reboot.send(RebootRequest());
else
tr->set(LiteralStringRef("\xff\xff/reboot_worker"),
address_interface[tokens[i]].first);
}
printf("Attempted to kill %zu processes\n", tokens.size() - 1);
}
@ -3391,6 +3415,69 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
continue;
}
if (tokencmp(tokens[0], "suspend")) {
getTransaction(db, tr, options, intrans);
if (tokens.size() == 1) {
Standalone<RangeResultRef> kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
std::vector<Future<Void>> addInterfs;
for( auto it : kvs ) {
addInterfs.push_back(addInterface(&address_interface, connectLock, it));
}
wait( waitForAll(addInterfs) );
if(address_interface.size() == 0) {
printf("\nNo addresses can be suspended.\n");
} else if(address_interface.size() == 1) {
printf("\nThe following address can be suspended:\n");
} else {
printf("\nThe following %zu addresses can be suspended:\n", address_interface.size());
}
for( auto it : address_interface ) {
printf("%s\n", printable(it.first).c_str());
}
printf("\n");
} else if(tokens.size() == 2) {
printUsage(tokens[0]);
is_error = true;
} else {
for(int i = 2; i < tokens.size(); i++) {
if(!address_interface.count(tokens[i])) {
printf("ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str());
is_error = true;
break;
}
}
if(!is_error) {
double seconds;
int n=0;
auto secondsStr = tokens[1].toString();
if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) {
printUsage(tokens[0]);
is_error = true;
} else {
int64_t timeout_ms = seconds*1000;
tr->setOption(FDBTransactionOptions::TIMEOUT, StringRef((uint8_t *)&timeout_ms, sizeof(int64_t)));
for(int i = 2; i < tokens.size(); i++) {
if (db->apiVersionAtLeast(700))
BinaryReader::fromStringRef<ClientWorkerInterface>(
address_interface[tokens[i]].first, IncludeVersion())
.reboot.send(RebootRequest(false, false, seconds));
else
tr->set(LiteralStringRef("\xff\xff/suspend_worker"),
address_interface[tokens[i]].first);
}
printf("Attempted to suspend %zu processes\n", tokens.size() - 2);
}
}
}
continue;
}
if (tokencmp(tokens[0], "force_recovery_with_data_loss")) {
if(tokens.size() != 2) {
printUsage(tokens[0]);
@ -3634,13 +3721,17 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
continue;
}
getTransaction(db, tr, options, intrans);
Standalone<RangeResultRef> kvs = wait(makeInterruptable(
tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces"),
LiteralStringRef("\xff\xff\xff")),
1)));
Standalone<RangeResultRef> kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
std::map<Key, ClientWorkerInterface> interfaces;
for (const auto& pair : kvs) {
auto ip_port = pair.key.endsWith(LiteralStringRef(":tls")) ? pair.key.removeSuffix(LiteralStringRef(":tls")) : pair.key;
auto ip_port = (pair.key.endsWith(LiteralStringRef(":tls"))
? pair.key.removeSuffix(LiteralStringRef(":tls"))
: pair.key)
.removePrefix(LiteralStringRef("\xff\xff/worker_interfaces/"));
interfaces.emplace(ip_port, BinaryReader::fromStringRef<ClientWorkerInterface>(pair.value, IncludeVersion()));
}
state Key ip_port = tokens[2];
@ -3665,7 +3756,11 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if (tokencmp(tokens[0], "expensive_data_check")) {
getTransaction(db, tr, options, intrans);
if (tokens.size() == 1) {
Standalone<RangeResultRef> kvs = wait( makeInterruptable( tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces"), LiteralStringRef("\xff\xff\xff")), 1) ) );
Standalone<RangeResultRef> kvs = wait(
makeInterruptable(tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY)));
ASSERT(!kvs.more);
Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
std::vector<Future<Void>> addInterfs;
for( auto it : kvs ) {
@ -3687,7 +3782,11 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
printf("\n");
} else if (tokencmp(tokens[1], "all")) {
for( auto it : address_interface ) {
tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), it.second.first);
if (db->apiVersionAtLeast(700))
BinaryReader::fromStringRef<ClientWorkerInterface>(it.second.first, IncludeVersion())
.reboot.send(RebootRequest(false, true));
else
tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), it.second.first);
}
if (address_interface.size() == 0) {
printf("ERROR: no processes to check. You must run the `expensive_data_check’ command before running `expensive_data_check all’.\n");
@ -3705,7 +3804,13 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if(!is_error) {
for(int i = 1; i < tokens.size(); i++) {
tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), address_interface[tokens[i]].first);
if (db->apiVersionAtLeast(700))
BinaryReader::fromStringRef<ClientWorkerInterface>(
address_interface[tokens[i]].first, IncludeVersion())
.reboot.send(RebootRequest(false, true));
else
tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"),
address_interface[tokens[i]].first);
}
printf("Attempted to kill and check %zu processes\n", tokens.size() - 1);
}
@ -4216,6 +4321,24 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
continue;
}
if (tokencmp(tokens[0], "cache_range")) {
if (tokens.size() != 4) {
printUsage(tokens[0]);
is_error = true;
continue;
}
KeyRangeRef cacheRange(tokens[2], tokens[3]);
if (tokencmp(tokens[1], "set")) {
wait(makeInterruptable(addCachedRange(db, cacheRange)));
} else if (tokencmp(tokens[1], "clear")) {
wait(makeInterruptable(removeCachedRange(db, cacheRange)));
} else {
printUsage(tokens[0]);
is_error = true;
}
continue;
}
printf("ERROR: Unknown command `%s'. Try `help'?\n", formatStringRef(tokens[0]).c_str());
is_error = true;

12
fdbclient/AsyncFileBlobStore.actor.h
View File

@ -59,7 +59,7 @@ public:
virtual void delref() { ReferenceCounted<AsyncFileBlobStoreWrite>::delref(); }
struct Part : ReferenceCounted<Part> {
Part(int n) : number(n), writer(content.getWriteBuffer(), NULL, Unversioned()), length(0) {
Part(int n, int minSize) : number(n), writer(content.getWriteBuffer(minSize), NULL, Unversioned()), length(0) {
etag = std::string();
::MD5_Init(&content_md5_buf);
}
@ -225,13 +225,13 @@ private:
// Do the upload, and if it fails forward errors to m_error and also stop if anything else sends an error to m_error
// Also, hold a releaser for the concurrent upload slot while all that is going on.
f->m_parts.back()->etag = holdWhile(std::shared_ptr<FlowLock::Releaser>(new FlowLock::Releaser(f->m_concurrentUploads, 1)),
joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error)
);
auto releaser = std::make_shared<FlowLock::Releaser>(f->m_concurrentUploads, 1);
f->m_parts.back()->etag =
holdWhile(std::move(releaser), joinErrorGroup(doPartUpload(f, f->m_parts.back().getPtr()), f->m_error));
// Make a new part to write to
if(startNew)
f->m_parts.push_back(Reference<Part>(new Part(f->m_parts.size() + 1)));
f->m_parts.push_back(Reference<Part>(new Part(f->m_parts.size() + 1, f->m_bstore->knobs.multipart_min_part_size)));
return Void();
}
@ -247,7 +247,7 @@ public:
: m_bstore(bstore), m_bucket(bucket), m_object(object), m_cursor(0), m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
// Add first part
m_parts.push_back(Reference<Part>(new Part(1)));
m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
}
};

4
fdbclient/BackupAgent.actor.h
View File

@ -929,8 +929,8 @@ struct StringRefReader {
// Functions for consuming big endian (network byte order) integers.
// Consumes a big endian number, swaps it to little endian, and returns it.
const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
const uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume<int32_t>()); }
uint32_t consumeNetworkUInt32() { return bigEndian32(consume<uint32_t>()); }
// Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value.
int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume<int64_t>()); }

14
fdbclient/BackupContainer.actor.cpp
View File

@ -842,10 +842,18 @@ public:
state std::vector<LogFile> logs;
state std::vector<LogFile> plogs;
TraceEvent("BackupContainerListFiles").detail("URL", bc->getURL());
wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) &&
store(plogs, bc->listLogFiles(scanBegin, scanEnd, true)) &&
store(desc.snapshots, bc->listKeyspaceSnapshots()));
TraceEvent("BackupContainerListFiles")
.detail("URL", bc->getURL())
.detail("LogFiles", logs.size())
.detail("PLogsFiles", plogs.size())
.detail("Snapshots", desc.snapshots.size());
if (plogs.size() > 0) {
desc.partitioned = true;
logs.swap(plogs);
@ -1207,7 +1215,7 @@ public:
}
// for each range in tags, check all tags from 1 are continouous
for (const auto [beginEnd, count] : tags) {
for (const auto& [beginEnd, count] : tags) {
for (int i = 1; i < count; i++) {
if (!isContinuous(files, tagIndices[i], beginEnd.first, std::min(beginEnd.second - 1, end), nullptr)) {
TraceEvent(SevWarn, "BackupFileNotContinuous")
@ -1310,7 +1318,7 @@ public:
// for each range in tags, check all partitions from 1 are continouous
Version lastEnd = begin;
for (const auto [beginEnd, count] : tags) {
for (const auto& [beginEnd, count] : tags) {
Version tagEnd = beginEnd.second; // This range's minimum continous partition version
for (int i = 1; i < count; i++) {
std::map<std::pair<Version, Version>, int> rangeTags;
@ -1629,7 +1637,7 @@ public:
std::string uniquePath = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".lnk";
unlink(uniquePath.c_str());
ASSERT(symlink(basename(path).c_str(), uniquePath.c_str()) == 0);
fullPath = uniquePath = uniquePath;
fullPath = uniquePath;
}
// Opening cached mode forces read/write mode at a lower level, overriding the readonly request. So cached mode
// can't be used because backup files are read-only. Cached mode can only help during restore task retries handled

4
fdbclient/BlobStore.actor.cpp
View File

@ -1057,7 +1057,7 @@ ACTOR Future<Void> writeEntireFileFromBuffer_impl(Reference<BlobStoreEndpoint> b
ACTOR Future<Void> writeEntireFile_impl(Reference<BlobStoreEndpoint> bstore, std::string bucket, std::string object, std::string content) {
state UnsentPacketQueue packets;
PacketWriter pw(packets.getWriteBuffer(), NULL, Unversioned());
PacketWriter pw(packets.getWriteBuffer(content.size()), NULL, Unversioned());
pw.serializeBytes(content);
if(content.size() > bstore->knobs.multipart_max_part_size)
throw file_too_large();
@ -1180,7 +1180,7 @@ ACTOR Future<Void> finishMultiPartUpload_impl(Reference<BlobStoreEndpoint> bstor
std::string resource = format("/%s/%s?uploadId=%s", bucket.c_str(), object.c_str(), uploadID.c_str());
HTTP::Headers headers;
PacketWriter pw(part_list.getWriteBuffer(), NULL, Unversioned());
PacketWriter pw(part_list.getWriteBuffer(manifest.size()), NULL, Unversioned());
pw.serializeBytes(manifest);
Reference<HTTP::Response> r = wait(bstore->doRequest("POST", resource, headers, &part_list, manifest.size(), {200}));
// TODO: In the event that the client times out just before the request completes (so the client is unaware) then the next retry

2
fdbclient/CMakeLists.txt
View File

@ -67,7 +67,7 @@ set(FDBCLIENT_SRCS
TagThrottle.h
TaskBucket.actor.cpp
TaskBucket.h
ThreadSafeTransaction.actor.cpp
ThreadSafeTransaction.cpp
ThreadSafeTransaction.h
Tuple.cpp
Tuple.h

2
fdbclient/ClientWorkerInterface.h
View File

@ -53,7 +53,7 @@ struct RebootRequest {
constexpr static FileIdentifier file_identifier = 11913957;
bool deleteData;
bool checkData;
uint32_t waitForDuration;
uint32_t waitForDuration; // seconds
explicit RebootRequest(bool deleteData = false, bool checkData = false, uint32_t waitForDuration = 0)
: deleteData(deleteData), checkData(checkData), waitForDuration(waitForDuration) {}

1
fdbclient/CommitTransaction.h
View File

@ -83,6 +83,7 @@ struct MutationRef {
MutationRef() {}
MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, Type t, StringRef a, StringRef b ) : type(t), param1(to, a), param2(to, b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); }

240
fdbclient/DatabaseConfiguration.cpp
View File

@ -224,111 +224,123 @@ bool DatabaseConfiguration::isValid() const {
StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
StatusObject result;
if( initialized ) {
std::string tlogInfo = tLogPolicy->info();
std::string storageInfo = storagePolicy->info();
bool customRedundancy = false;
if( tLogWriteAntiQuorum == 0 ) {
if( tLogReplicationFactor == 1 && storageTeamSize == 1 ) {
result["redundancy_mode"] = "single";
} else if( tLogReplicationFactor == 2 && storageTeamSize == 2 ) {
result["redundancy_mode"] = "double";
} else if( tLogReplicationFactor == 4 && storageTeamSize == 6 && tlogInfo == "dcid^2 x zoneid^2 x 1" && storageInfo == "dcid^3 x zoneid^2 x 1" ) {
result["redundancy_mode"] = "three_datacenter";
} else if( tLogReplicationFactor == 4 && storageTeamSize == 4 && tlogInfo == "dcid^2 x zoneid^2 x 1" && storageInfo == "dcid^2 x zoneid^2 x 1" ) {
result["redundancy_mode"] = "three_datacenter_fallback";
} else if( tLogReplicationFactor == 3 && storageTeamSize == 3 ) {
result["redundancy_mode"] = "triple";
} else if( tLogReplicationFactor == 4 && storageTeamSize == 3 && tlogInfo == "data_hall^2 x zoneid^2 x 1" && storageInfo == "data_hall^3 x 1" ) {
result["redundancy_mode"] = "three_data_hall";
} else if( tLogReplicationFactor == 4 && storageTeamSize == 2 && tlogInfo == "data_hall^2 x zoneid^2 x 1" && storageInfo == "data_hall^2 x 1" ) {
result["redundancy_mode"] = "three_data_hall_fallback";
} else {
customRedundancy = true;
}
if (!initialized) {
return result;
}
std::string tlogInfo = tLogPolicy->info();
std::string storageInfo = storagePolicy->info();
bool customRedundancy = false;
if (tLogWriteAntiQuorum == 0) {
if (tLogReplicationFactor == 1 && storageTeamSize == 1) {
result["redundancy_mode"] = "single";
} else if (tLogReplicationFactor == 2 && storageTeamSize == 2) {
result["redundancy_mode"] = "double";
} else if (tLogReplicationFactor == 4 && storageTeamSize == 6 && tlogInfo == "dcid^2 x zoneid^2 x 1" &&
storageInfo == "dcid^3 x zoneid^2 x 1") {
result["redundancy_mode"] = "three_datacenter";
} else if (tLogReplicationFactor == 4 && storageTeamSize == 4 && tlogInfo == "dcid^2 x zoneid^2 x 1" &&
storageInfo == "dcid^2 x zoneid^2 x 1") {
result["redundancy_mode"] = "three_datacenter_fallback";
} else if (tLogReplicationFactor == 3 && storageTeamSize == 3) {
result["redundancy_mode"] = "triple";
} else if (tLogReplicationFactor == 4 && storageTeamSize == 3 && tlogInfo == "data_hall^2 x zoneid^2 x 1" &&
storageInfo == "data_hall^3 x 1") {
result["redundancy_mode"] = "three_data_hall";
} else if (tLogReplicationFactor == 4 && storageTeamSize == 2 && tlogInfo == "data_hall^2 x zoneid^2 x 1" &&
storageInfo == "data_hall^2 x 1") {
result["redundancy_mode"] = "three_data_hall_fallback";
} else {
customRedundancy = true;
}
if(customRedundancy) {
result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum;
if(!noPolicies) result["storage_replication_policy"] = storagePolicy->info();
if(!noPolicies) result["log_replication_policy"] = tLogPolicy->info();
}
if ( tLogVersion > TLogVersion::DEFAULT ) {
result["log_version"] = (int)tLogVersion;
}
if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V1 && storageServerStoreType == KeyValueStoreType::SSD_BTREE_V1) {
result["storage_engine"] = "ssd-1";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::SSD_BTREE_V2) {
result["storage_engine"] = "ssd-2";
} else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::SSD_REDWOOD_V1 ) {
result["storage_engine"] = "ssd-redwood-experimental";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
result["storage_engine"] = "ssd-rocksdb-experimental";
} else if( tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY ) {
result["storage_engine"] = "memory-1";
} else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::MEMORY_RADIXTREE ) {
result["storage_engine"] = "memory-radixtree-beta";
} else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::MEMORY ) {
result["storage_engine"] = "memory-2";
} else {
result["storage_engine"] = "custom";
}
result["log_spill"] = (int)tLogSpillType;
if( remoteTLogReplicationFactor == 1 ) {
result["remote_redundancy_mode"] = "remote_single";
} else if( remoteTLogReplicationFactor == 2 ) {
result["remote_redundancy_mode"] = "remote_double";
} else if( remoteTLogReplicationFactor == 3 ) {
result["remote_redundancy_mode"] = "remote_triple";
} else if( remoteTLogReplicationFactor > 3 ) {
result["remote_log_replicas"] = remoteTLogReplicationFactor;
if(noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info();
}
result["usable_regions"] = usableRegions;
if(regions.size()) {
result["regions"] = getRegionJSON();
}
if( desiredTLogCount != -1 ) {
result["logs"] = desiredTLogCount;
}
if( masterProxyCount != -1 ) {
result["proxies"] = masterProxyCount;
}
if( resolverCount != -1 ) {
result["resolvers"] = resolverCount;
}
if( desiredLogRouterCount != -1 ) {
result["log_routers"] = desiredLogRouterCount;
}
if( remoteDesiredTLogCount != -1 ) {
result["remote_logs"] = remoteDesiredTLogCount;
}
if( repopulateRegionAntiQuorum != 0 ) {
result["repopulate_anti_quorum"] = repopulateRegionAntiQuorum;
}
if( autoMasterProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_PROXIES ) {
result["auto_proxies"] = autoMasterProxyCount;
}
if (autoResolverCount != CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS) {
result["auto_resolvers"] = autoResolverCount;
}
if (autoDesiredTLogCount != CLIENT_KNOBS->DEFAULT_AUTO_LOGS) {
result["auto_logs"] = autoDesiredTLogCount;
}
result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
} else {
customRedundancy = true;
}
if (customRedundancy) {
result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info();
}
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
result["log_version"] = (int)tLogVersion;
}
if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V1 &&
storageServerStoreType == KeyValueStoreType::SSD_BTREE_V1) {
result["storage_engine"] = "ssd-1";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
storageServerStoreType == KeyValueStoreType::SSD_BTREE_V2) {
result["storage_engine"] = "ssd-2";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
storageServerStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
result["storage_engine"] = "ssd-redwood-experimental";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
result["storage_engine"] = "ssd-rocksdb-experimental";
} else if (tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY) {
result["storage_engine"] = "memory-1";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
storageServerStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
result["storage_engine"] = "memory-radixtree-beta";
} else if (tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 &&
storageServerStoreType == KeyValueStoreType::MEMORY) {
result["storage_engine"] = "memory-2";
} else {
result["storage_engine"] = "custom";
}
result["log_spill"] = (int)tLogSpillType;
if (remoteTLogReplicationFactor == 1) {
result["remote_redundancy_mode"] = "remote_single";
} else if (remoteTLogReplicationFactor == 2) {
result["remote_redundancy_mode"] = "remote_double";
} else if (remoteTLogReplicationFactor == 3) {
result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info();
}
result["usable_regions"] = usableRegions;
if (regions.size()) {
result["regions"] = getRegionJSON();
}
if (desiredTLogCount != -1 || isOverridden("logs")) {
result["logs"] = desiredTLogCount;
}
if (masterProxyCount != -1 || isOverridden("proxies")) {
result["proxies"] = masterProxyCount;
}
if (resolverCount != -1 || isOverridden("resolvers")) {
result["resolvers"] = resolverCount;
}
if (desiredLogRouterCount != -1 || isOverridden("log_routers")) {
result["log_routers"] = desiredLogRouterCount;
}
if (remoteDesiredTLogCount != -1 || isOverridden("remote_logs")) {
result["remote_logs"] = remoteDesiredTLogCount;
}
if (repopulateRegionAntiQuorum != 0 || isOverridden("repopulate_anti_quorum")) {
result["repopulate_anti_quorum"] = repopulateRegionAntiQuorum;
}
if (autoMasterProxyCount != CLIENT_KNOBS->DEFAULT_AUTO_PROXIES || isOverridden("auto_proxies")) {
result["auto_proxies"] = autoMasterProxyCount;
}
if (autoResolverCount != CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS || isOverridden("auto_resolvers")) {
result["auto_resolvers"] = autoResolverCount;
}
if (autoDesiredTLogCount != CLIENT_KNOBS->DEFAULT_AUTO_LOGS || isOverridden("auto_logs")) {
result["auto_logs"] = autoDesiredTLogCount;
}
result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
return result;
}
@ -540,3 +552,31 @@ void DatabaseConfiguration::makeConfigurationImmutable() {
rawConfiguration[i++] = KeyValueRef( rawConfiguration.arena(), KeyValueRef( r->first, r->second ) );
mutableConfiguration = Optional<std::map<std::string,std::string>>();
}
void DatabaseConfiguration::fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig) {
resetInternal();
this->rawConfiguration = rawConfig;
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) {
setInternal(c->key, c->value);
}
setDefaultReplicationPolicy();
}
bool DatabaseConfiguration::isOverridden(std::string key) const {
key = configKeysPrefix.toString() + key;
if (mutableConfiguration.present()) {
return mutableConfiguration.get().find(key) != mutableConfiguration.get().end();
}
const int keyLen = key.size();
for (auto iter = rawConfiguration.begin(); iter != rawConfiguration.end(); ++iter) {
const auto& rawConfKey = iter->key;
if (keyLen == rawConfKey.size() &&
strncmp(key.c_str(), reinterpret_cast<const char*>(rawConfKey.begin()), keyLen) == 0) {
return true;
}
}
return false;
}

11
fdbclient/DatabaseConfiguration.h
View File

@ -219,13 +219,7 @@ struct DatabaseConfiguration {
}
}
void fromKeyValues( Standalone<VectorRef<KeyValueRef>> rawConfig ) {
resetInternal();
this->rawConfiguration = rawConfig;
for(auto c=rawConfiguration.begin(); c!=rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy();
}
void fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig);
private:
Optional< std::map<std::string, std::string> > mutableConfiguration; // If present, rawConfiguration is not valid
@ -237,6 +231,9 @@ private:
bool setInternal( KeyRef key, ValueRef value );
void resetInternal();
void setDefaultReplicationPolicy();
/// Check if the key is overridden by either mutableConfiguration or rawConfiguration
bool isOverridden(std::string key) const;
};
#endif

6
fdbclient/DatabaseContext.h
View File

@ -339,11 +339,13 @@ public:
double detailedHealthMetricsLastUpdated;
UniqueOrderedOptionList<FDBTransactionOptions> transactionDefaults;
Future<Void> cacheListMonitor;
AsyncTrigger updateCache;
std::vector<std::unique_ptr<SpecialKeyRangeBaseImpl>> specialKeySpaceModules;
std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules;
std::unique_ptr<SpecialKeySpace> specialKeySpace;
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, std::unique_ptr<SpecialKeyRangeBaseImpl> impl);
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl> &&impl);
static bool debugUseTags;
static const std::vector<std::string> debugTransactionTagChoices;

6
fdbclient/FDBTypes.h
View File

@ -749,11 +749,11 @@ struct TLogVersion {
V2 = 2, // 6.0
V3 = 3, // 6.1
V4 = 4, // 6.2
V5 = 5, // 7.0
V5 = 5, // 6.3
MIN_SUPPORTED = V2,
MAX_SUPPORTED = V5,
MIN_RECRUITABLE = V3,
DEFAULT = V4,
MIN_RECRUITABLE = V4,
DEFAULT = V5,
} version;
TLogVersion() : version(UNSET) {}

6
fdbclient/HTTP.actor.cpp
View File

@ -352,9 +352,6 @@ namespace HTTP {
send_start = timer();
loop {
wait(conn->onWritable());
wait( delay( 0, TaskPriority::WriteSocket ) );
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
@ -375,6 +372,9 @@ namespace HTTP {
pContent->sent(len);
if(pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);

2
fdbclient/Knobs.cpp
View File

@ -89,7 +89,7 @@ void ClientKnobs::initialize(bool randomize) {
init( STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, 15.0 );
init( AGGREGATE_HEALTH_METRICS_MAX_STALENESS, 0.5 );
init( DETAILED_HEALTH_METRICS_MAX_STALENESS, 5.0 );
init( TAG_ENCODE_KEY_SERVERS, false ); if( randomize && BUGGIFY ) TAG_ENCODE_KEY_SERVERS = true;
init( TAG_ENCODE_KEY_SERVERS, true ); if( randomize && BUGGIFY ) TAG_ENCODE_KEY_SERVERS = false;
//KeyRangeMap
init( KRM_GET_RANGE_LIMIT, 1e5 ); if( randomize && BUGGIFY ) KRM_GET_RANGE_LIMIT = 10;

307
fdbclient/ManagementAPI.actor.cpp
View File

@ -38,7 +38,6 @@
#include "fdbrpc/Replication.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR static Future<vector<AddressExclusion>> getExcludedServers(Transaction* tr);
bool isInteger(const std::string& s) {
if( s.empty() ) return false;
@ -1264,93 +1263,152 @@ struct AutoQuorumChange : IQuorumChange {
};
Reference<IQuorumChange> autoQuorumChange( int desired ) { return Reference<IQuorumChange>(new AutoQuorumChange(desired)); }
void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed) {
tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE );
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
tr.setOption( FDBTransactionOptions::USE_PROVISIONAL_PROXIES );
std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
auto serversVersionKey = failed ? failedServersVersionKey : excludedServersVersionKey;
tr.addReadConflictRange( singleKeyRange(serversVersionKey) ); //To conflict with parallel includeServers
tr.set( serversVersionKey, excludeVersionKey );
for(auto& s : servers) {
if (failed) {
tr.set( encodeFailedServersKey(s), StringRef() );
} else {
tr.set( encodeExcludedServersKey(s), StringRef() );
}
}
TraceEvent("ExcludeServersCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed);
}
ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers, bool failed) {
state Transaction tr(cx);
state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(),Unversioned());
state std::string excludeVersionKey = deterministicRandom()->randomUniqueID().toString();
loop {
try {
tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE );
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
tr.setOption( FDBTransactionOptions::USE_PROVISIONAL_PROXIES );
auto serversVersionKey = failed ? failedServersVersionKey : excludedServersVersionKey;
tr.addReadConflictRange( singleKeyRange(serversVersionKey) ); //To conflict with parallel includeServers
tr.set( serversVersionKey, excludeVersionKey );
for(auto& s : servers) {
if (failed) {
tr.set( encodeFailedServersKey(s), StringRef() );
} else {
tr.set( encodeExcludedServersKey(s), StringRef() );
if (cx->apiVersionAtLeast(700)) {
state ReadYourWritesTransaction ryw(cx);
loop {
try{
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw.set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"), ValueRef());
for(auto& s : servers) {
Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.set(addr, ValueRef());
}
TraceEvent("ExcludeServersSpecialKeySpaceCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed);
wait(ryw.commit());
return Void();
} catch (Error& e) {
wait( ryw.onError(e) );
}
}
} else {
state Transaction tr(cx);
loop {
try {
excludeServers(tr, servers, failed);
wait( tr.commit() );
return Void();
} catch (Error& e) {
wait( tr.onError(e) );
}
TraceEvent("ExcludeServersCommit").detail("Servers", describe(servers)).detail("ExcludeFailed", failed);
wait( tr.commit() );
return Void();
} catch (Error& e) {
wait( tr.onError(e) );
}
}
}
ACTOR Future<Void> includeServers(Database cx, vector<AddressExclusion> servers, bool failed) {
state Transaction tr(cx);
state std::string versionKey = deterministicRandom()->randomUniqueID().toString();
loop {
try {
tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE );
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
tr.setOption( FDBTransactionOptions::USE_PROVISIONAL_PROXIES );
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY
tr.setOption( FDBTransactionOptions::CAUSAL_WRITE_RISKY );
if (failed) {
tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
tr.set(failedServersVersionKey, versionKey);
} else {
tr.addReadConflictRange(singleKeyRange(excludedServersVersionKey));
tr.set(excludedServersVersionKey, versionKey);
}
for(auto& s : servers ) {
if (!s.isValid()) {
if (failed) {
tr.clear(failedServersKeys);
if (cx->apiVersionAtLeast(700)) {
state ReadYourWritesTransaction ryw(cx);
loop {
try {
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (auto& s : servers) {
if (!s.isValid()) {
if (failed) {
ryw.clear(SpecialKeySpace::getManamentApiCommandRange("failed"));
} else {
ryw.clear(SpecialKeySpace::getManamentApiCommandRange("exclude"));
}
} else {
tr.clear(excludedServersKeys);
}
} else if (s.isWholeMachine()) {
// Eliminate both any ip-level exclusion (1.2.3.4) and any
// port-level exclusions (1.2.3.4:5)
// The range ['IP', 'IP;'] was originally deleted. ';' is
// char(':' + 1). This does not work, as other for all
// x between 0 and 9, 'IPx' will also be in this range.
//
// This is why we now make two clears: first only of the ip
// address, the second will delete all ports.
auto addr = failed ? encodeFailedServersKey(s) : encodeExcludedServersKey(s);
tr.clear(singleKeyRange(addr));
tr.clear(KeyRangeRef(addr + ':', addr + char(':' + 1)));
} else {
if (failed) {
tr.clear(encodeFailedServersKey(s));
} else {
tr.clear(encodeExcludedServersKey(s));
Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.clear(addr);
// Eliminate both any ip-level exclusion (1.2.3.4) and any
// port-level exclusions (1.2.3.4:5)
// The range ['IP', 'IP;'] was originally deleted. ';' is
// char(':' + 1). This does not work, as other for all
// x between 0 and 9, 'IPx' will also be in this range.
//
// This is why we now make two clears: first only of the ip
// address, the second will delete all ports.
if (s.isWholeMachine())
ryw.clear(KeyRangeRef(addr.withSuffix(LiteralStringRef(":")), addr.withSuffix(LiteralStringRef(";"))));
}
}
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
wait( ryw.commit() );
return Void();
} catch (Error& e) {
TraceEvent("IncludeServersError").error(e, true);
wait( ryw.onError(e) );
}
}
} else {
state Transaction tr(cx);
loop {
try {
tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE );
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
tr.setOption( FDBTransactionOptions::USE_PROVISIONAL_PROXIES );
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and CAUSAL_WRITE_RISKY
tr.setOption( FDBTransactionOptions::CAUSAL_WRITE_RISKY );
if (failed) {
tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
tr.set(failedServersVersionKey, versionKey);
} else {
tr.addReadConflictRange(singleKeyRange(excludedServersVersionKey));
tr.set(excludedServersVersionKey, versionKey);
}
wait( tr.commit() );
return Void();
} catch (Error& e) {
TraceEvent("IncludeServersError").error(e, true);
wait( tr.onError(e) );
for(auto& s : servers ) {
if (!s.isValid()) {
if (failed) {
tr.clear(failedServersKeys);
} else {
tr.clear(excludedServersKeys);
}
} else if (s.isWholeMachine()) {
// Eliminate both any ip-level exclusion (1.2.3.4) and any
// port-level exclusions (1.2.3.4:5)
// The range ['IP', 'IP;'] was originally deleted. ';' is
// char(':' + 1). This does not work, as other for all
// x between 0 and 9, 'IPx' will also be in this range.
//
// This is why we now make two clears: first only of the ip
// address, the second will delete all ports.
auto addr = failed ? encodeFailedServersKey(s) : encodeExcludedServersKey(s);
tr.clear(singleKeyRange(addr));
tr.clear(KeyRangeRef(addr + ':', addr + char(':' + 1)));
} else {
if (failed) {
tr.clear(encodeFailedServersKey(s));
} else {
tr.clear(encodeExcludedServersKey(s));
}
}
}
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
wait( tr.commit() );
return Void();
} catch (Error& e) {
TraceEvent("IncludeServersError").error(e, true);
wait( tr.onError(e) );
}
}
}
}
@ -1389,7 +1447,7 @@ ACTOR Future<Void> setClass( Database cx, AddressExclusion server, ProcessClass
}
}
ACTOR static Future<vector<AddressExclusion>> getExcludedServers( Transaction* tr ) {
ACTOR Future<vector<AddressExclusion>> getExcludedServers( Transaction* tr ) {
state Standalone<RangeResultRef> r = wait( tr->getRange( excludedServersKeys, CLIENT_KNOBS->TOO_MANY ) );
ASSERT( !r.more && r.size() < CLIENT_KNOBS->TOO_MANY );
state Standalone<RangeResultRef> r2 = wait( tr->getRange( failedServersKeys, CLIENT_KNOBS->TOO_MANY ) );
@ -1559,59 +1617,67 @@ ACTOR Future<int> setDDMode( Database cx, int mode ) {
}
}
ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
std::set<AddressExclusion>* exclusions,
std::set<NetworkAddress>* inProgressExclusion) {
// TODO : replace using ExclusionInProgressRangeImpl in special key space
ASSERT(inProgressExclusion->size() == 0); // Make sure every time it is cleared beforehand
if (!exclusions->size()) return true;
tr->setOption( FDBTransactionOptions::READ_SYSTEM_KEYS );
tr->setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE ); // necessary?
tr->setOption( FDBTransactionOptions::LOCK_AWARE );
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed recovery
// Check that there aren't any storage servers with addresses violating the exclusions
Standalone<RangeResultRef> serverList = wait( tr->getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) );
ASSERT( !serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY );
state bool ok = true;
for(auto& s : serverList) {
auto addresses = decodeServerListValue( s.value ).getKeyValues.getEndpoint().addresses;
if ( addressExcluded(*exclusions, addresses.address) ) {
ok = false;
inProgressExclusion->insert(addresses.address);
}
if ( addresses.secondaryAddress.present() && addressExcluded(*exclusions, addresses.secondaryAddress.get()) ) {
ok = false;
inProgressExclusion->insert(addresses.secondaryAddress.get());
}
}
if (ok) {
Optional<Standalone<StringRef>> value = wait( tr->get(logsKey) );
ASSERT(value.present());
auto logs = decodeLogsValue(value.get());
for( auto const& log : logs.first ) {
if (log.second == NetworkAddress() || addressExcluded(*exclusions, log.second)) {
ok = false;
inProgressExclusion->insert(log.second);
}
}
for( auto const& log : logs.second ) {
if (log.second == NetworkAddress() || addressExcluded(*exclusions, log.second)) {
ok = false;
inProgressExclusion->insert(log.second);
}
}
}
return ok;
}
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> excl,
bool waitForAllExcluded) {
state std::set<AddressExclusion> exclusions( excl.begin(), excl.end() );
state std::set<NetworkAddress> inProgressExclusion;
if (!excl.size()) return inProgressExclusion;
loop {
state Transaction tr(cx);
state ReadYourWritesTransaction tr(cx);
inProgressExclusion.clear();
try {
tr.setOption( FDBTransactionOptions::READ_SYSTEM_KEYS );
tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE ); // necessary?
tr.setOption( FDBTransactionOptions::LOCK_AWARE );
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed recovery
// Check that there aren't any storage servers with addresses violating the exclusions
Standalone<RangeResultRef> serverList = wait( tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) );
ASSERT( !serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY );
state bool ok = true;
inProgressExclusion.clear();
for(auto& s : serverList) {
auto addresses = decodeServerListValue( s.value ).getKeyValues.getEndpoint().addresses;
if ( addressExcluded(exclusions, addresses.address) ) {
ok = false;
inProgressExclusion.insert(addresses.address);
}
if ( addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get()) ) {
ok = false;
inProgressExclusion.insert(addresses.secondaryAddress.get());
}
}
if (ok) {
Optional<Standalone<StringRef>> value = wait( tr.get(logsKey) );
ASSERT(value.present());
auto logs = decodeLogsValue(value.get());
for( auto const& log : logs.first ) {
if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
ok = false;
inProgressExclusion.insert(log.second);
}
}
for( auto const& log : logs.second ) {
if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
ok = false;
inProgressExclusion.insert(log.second);
}
}
}
bool ok = wait(checkForExcludingServersTxActor(&tr, &exclusions, &inProgressExclusion));
if (ok) return inProgressExclusion;
if (!waitForAllExcluded) break;
@ -1620,7 +1686,6 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vec
wait( tr.onError(e) );
}
}
return inProgressExclusion;
}

5
fdbclient/ManagementAPI.actor.h
View File

@ -144,6 +144,7 @@ Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQu
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without necessarily waiting for
// the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the given IP.
ACTOR Future<Void> excludeServers( Database cx, vector<AddressExclusion> servers, bool failed = false );
void excludeServers(Transaction& tr, vector<AddressExclusion>& servers, bool failed = false);
// Remove the given servers from the exclusion list. A NetworkAddress with a port of 0 means all servers on the given IP. A NetworkAddress() means
// all servers (don't exclude anything)
@ -154,12 +155,16 @@ ACTOR Future<Void> setClass( Database cx, AddressExclusion server, ProcessClas
// Get the current list of excluded servers
ACTOR Future<vector<AddressExclusion>> getExcludedServers( Database cx );
ACTOR Future<vector<AddressExclusion>> getExcludedServers( Transaction* tr);
// Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is
// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and
// unless any of them are explicitly included with includeServers()
ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Database cx, vector<AddressExclusion> servers,
bool waitForAllExcluded);
ACTOR Future<bool> checkForExcludingServersTxActor(ReadYourWritesTransaction* tr,
std::set<AddressExclusion>* exclusions,
std::set<NetworkAddress>* inProgressExclusion);
// Gets a list of all workers in the cluster (excluding testers)
ACTOR Future<vector<ProcessData>> getWorkers( Database cx );

15
fdbclient/MasterProxyInterface.h
View File

@ -50,7 +50,6 @@ struct MasterProxyInterface {
RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream< struct GetRawCommittedVersionRequest > getRawCommittedVersion;
RequestStream< struct TxnStateRequest > txnState;
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics;
RequestStream< struct ProxySnapRequest > proxySnapReq;
@ -71,12 +70,11 @@ struct MasterProxyInterface {
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) );
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) );
waitFailure = RequestStream<ReplyPromise<Void>>( commit.getEndpoint().getAdjustedEndpoint(4) );
getRawCommittedVersion = RequestStream< struct GetRawCommittedVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(5) );
txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(6) );
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(7) );
proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(8) );
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(9) );
getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(10) );
txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(5) );
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(6) );
proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(7) );
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(8) );
getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(9) );
}
}
@ -87,7 +85,6 @@ struct MasterProxyInterface {
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver());
streams.push_back(getRawCommittedVersion.getReceiver(TaskPriority::ProxyGetRawCommittedVersion));
streams.push_back(txnState.getReceiver());
streams.push_back(getHealthMetrics.getReceiver());
streams.push_back(proxySnapReq.getReceiver());
@ -435,7 +432,7 @@ struct ProxySnapRequest
{
constexpr static FileIdentifier file_identifier = 5427684;
Arena arena;
StringRef snapPayload;
StringRef snapPayload; // command used to snapshot the data folder
UID snapUID;
ReplyPromise<Void> reply;
Optional<UID> debugID;

26
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -767,15 +767,16 @@ void MultiVersionDatabase::Connector::connect() {
}
tr = candidateDatabase->createTransaction();
return ErrorOr<ThreadFuture<Void>>(mapThreadFuture<Version, Void>(tr->getReadVersion(), [this](ErrorOr<Version> v) {
// If the version attempt returns an error, we regard that as a connection (except operation_cancelled)
if(v.isError() && v.getError().code() == error_code_operation_cancelled) {
return ErrorOr<Void>(v.getError());
}
else {
return ErrorOr<Void>(Void());
}
}));
return ErrorOr<ThreadFuture<Void>>(
mapThreadFuture<Version, Void>(tr->getReadVersion(), [](ErrorOr<Version> v) {
// If the version attempt returns an error, we regard that as a connection (except
// operation_cancelled)
if (v.isError() && v.getError().code() == error_code_operation_cancelled) {
return ErrorOr<Void>(v.getError());
} else {
return ErrorOr<Void>(Void());
}
}));
});
@ -1045,7 +1046,7 @@ void MultiVersionApi::setSupportedClientVersions(Standalone<StringRef> versions)
}, NULL);
if(!bypassMultiClientApi) {
runOnExternalClients([this, versions](Reference<ClientInfo> client){
runOnExternalClients([versions](Reference<ClientInfo> client) {
client->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions);
});
}
@ -1105,9 +1106,8 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
if(!bypassMultiClientApi) {
if(networkSetup) {
runOnExternalClients([this, option, value](Reference<ClientInfo> client) {
client->api->setNetworkOption(option, value);
});
runOnExternalClients(
[option, value](Reference<ClientInfo> client) { client->api->setNetworkOption(option, value); });
}
else {
options.push_back(std::make_pair(option, value.castTo<Standalone<StringRef>>()));

138
fdbclient/NativeAPI.actor.cpp
View File

@ -696,15 +696,16 @@ Future<HealthMetrics> DatabaseContext::getHealthMetrics(bool detailed = false) {
return getHealthMetricsActor(this, detailed);
}
void DatabaseContext::registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, std::unique_ptr<SpecialKeyRangeBaseImpl> impl) {
specialKeySpace->registerKeyRange(module, impl->getKeyRange(), impl.get());
void DatabaseContext::registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl> &&impl) {
specialKeySpace->registerKeyRange(module, type, impl->getKeyRange(), impl.get());
specialKeySpaceModules.push_back(std::move(impl));
}
ACTOR Future<Standalone<RangeResultRef>> getWorkerInterfaces(Reference<ClusterConnectionFile> clusterFile);
ACTOR Future<Optional<Value>> getJSON(Database db);
struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeBaseImpl {
struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeReadImpl {
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override {
if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionFile()) {
Key prefix = Key(getKeyRange().begin);
@ -724,10 +725,10 @@ struct WorkerInterfacesSpecialKeyImpl : SpecialKeyRangeBaseImpl {
}
}
explicit WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {}
explicit WorkerInterfacesSpecialKeyImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
};
struct SingleSpecialKeyImpl : SpecialKeyRangeBaseImpl {
struct SingleSpecialKeyImpl : SpecialKeyRangeReadImpl {
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override {
ASSERT(kr.contains(k));
return map(f(ryw), [k = k](Optional<Value> v) {
@ -740,7 +741,7 @@ struct SingleSpecialKeyImpl : SpecialKeyRangeBaseImpl {
}
SingleSpecialKeyImpl(KeyRef k, const std::function<Future<Optional<Value>>(ReadYourWritesTransaction*)>& f)
: SpecialKeyRangeBaseImpl(singleKeyRange(k)), k(k), f(f) {}
: SpecialKeyRangeReadImpl(singleKeyRange(k)), k(k), f(f) {}
private:
Key k;
@ -846,7 +847,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
: connectionFile(connectionFile), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), taskID(taskID),
clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware),
apiVersion(apiVersion), switchable(switchable), provisional(false), cc("TransactionMetrics"),
transactionReadVersions("ReadVersions", cc),
transactionReadVersions("ReadVersions", cc),
transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc),
transactionReadVersionBatches("ReadVersionBatches", cc),
@ -894,20 +895,52 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
monitorMasterProxiesInfoChange = monitorMasterProxiesChange(clientInfo, &masterProxiesChangeTrigger);
clientStatusUpdater.actor = clientStatusUpdateActor(this);
cacheListMonitor = monitorCacheList(this);
if (apiVersionAtLeast(700)) {
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::ERRORMSG, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<SingleSpecialKeyImpl>(
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin,
[](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
if (ryw->getSpecialKeySpaceErrorMsg().present())
return Optional<Value>(ryw->getSpecialKeySpaceErrorMsg().get());
else
return Optional<Value>();
}));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE,
std::make_unique<ManagementCommandsOptionsImpl>(
KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE,
std::make_unique<ExcludeServersRangeImpl>(SpecialKeySpace::getManamentApiCommandRange("exclude")));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READWRITE,
std::make_unique<FailedServersRangeImpl>(SpecialKeySpace::getManamentApiCommandRange("failed")));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::MANAGEMENT, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<ExclusionInProgressRangeImpl>(
KeyRangeRef(LiteralStringRef("inProgressExclusion/"), LiteralStringRef("inProgressExclusion0"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)));
}
if (apiVersionAtLeast(630)) {
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, std::make_unique<ConflictingKeysImpl>(conflictingKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, std::make_unique<ReadConflictRangeImpl>(readConflictRangeKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, std::make_unique<WriteConflictRangeImpl>(writeConflictRangeKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::METRICS,
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<ConflictingKeysImpl>(conflictingKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<ReadConflictRangeImpl>(readConflictRangeKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::TRANSACTION, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<WriteConflictRangeImpl>(writeConflictRangeKeysRange));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::METRICS, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<DDStatsRangeImpl>(ddStatsRange));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::METRICS,
SpecialKeySpace::MODULE::METRICS, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<HealthMetricsRangeImpl>(KeyRangeRef(LiteralStringRef("\xff\xff/metrics/health/"),
LiteralStringRef("\xff\xff/metrics/health0"))));
registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::WORKERINTERFACE, std::make_unique<WorkerInterfacesSpecialKeyImpl>(KeyRangeRef(
LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0"))));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::STATUSJSON,
SpecialKeySpace::MODULE::WORKERINTERFACE, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<WorkerInterfacesSpecialKeyImpl>(KeyRangeRef(
LiteralStringRef("\xff\xff/worker_interfaces/"), LiteralStringRef("\xff\xff/worker_interfaces0"))));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::STATUSJSON, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<SingleSpecialKeyImpl>(LiteralStringRef("\xff\xff/status/json"),
[](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
if (ryw->getDatabase().getPtr() &&
@ -918,7 +951,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
}
}));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::CLUSTERFILEPATH,
SpecialKeySpace::MODULE::CLUSTERFILEPATH, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<SingleSpecialKeyImpl>(
LiteralStringRef("\xff\xff/cluster_file_path"),
[](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
@ -934,7 +967,7 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
}));
registerSpecialKeySpaceModule(
SpecialKeySpace::MODULE::CONNECTIONSTRING,
SpecialKeySpace::MODULE::CONNECTIONSTRING, SpecialKeySpace::IMPLTYPE::READONLY,
std::make_unique<SingleSpecialKeyImpl>(
LiteralStringRef("\xff\xff/connection_string"),
[](ReadYourWritesTransaction* ryw) -> Future<Optional<Value>> {
@ -958,19 +991,19 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionF
}
DatabaseContext::DatabaseContext( const Error &err ) : deferredError(err), cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc), transactionReadVersionsThrottled("ReadVersionsThrottled", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc), transactionReadVersionBatches("ReadVersionBatches", cc), transactionBatchReadVersions("BatchPriorityReadVersions", cc),
transactionDefaultReadVersions("DefaultPriorityReadVersions", cc), transactionImmediateReadVersions("ImmediatePriorityReadVersions", cc),
transactionBatchReadVersionsCompleted("BatchPriorityReadVersionsCompleted", cc), transactionDefaultReadVersionsCompleted("DefaultPriorityReadVersionsCompleted", cc),
transactionImmediateReadVersionsCompleted("ImmediatePriorityReadVersionsCompleted", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc),
transactionPhysicalReadsCompleted("PhysicalReadRequestsCompleted", cc), transactionGetKeyRequests("GetKeyRequests", cc), transactionGetValueRequests("GetValueRequests", cc),
transactionGetRangeRequests("GetRangeRequests", cc), transactionWatchRequests("WatchRequests", cc), transactionGetAddressesForKeyRequests("GetAddressesForKeyRequests", cc),
transactionBytesRead("BytesRead", cc), transactionKeysRead("KeysRead", cc), transactionMetadataVersionReads("MetadataVersionReads", cc), transactionCommittedMutations("CommittedMutations", cc),
transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionSetMutations("SetMutations", cc), transactionClearMutations("ClearMutations", cc),
transactionAtomicMutations("AtomicMutations", cc), transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc),
transactionKeyServerLocationRequests("KeyServerLocationRequests", cc), transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc), transactionsTooOld("TooOld", cc),
transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc),
transactionReadVersionsCompleted("ReadVersionsCompleted", cc), transactionReadVersionBatches("ReadVersionBatches", cc), transactionBatchReadVersions("BatchPriorityReadVersions", cc),
transactionDefaultReadVersions("DefaultPriorityReadVersions", cc), transactionImmediateReadVersions("ImmediatePriorityReadVersions", cc),
transactionBatchReadVersionsCompleted("BatchPriorityReadVersionsCompleted", cc), transactionDefaultReadVersionsCompleted("DefaultPriorityReadVersionsCompleted", cc),
transactionImmediateReadVersionsCompleted("ImmediatePriorityReadVersionsCompleted", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc),
transactionPhysicalReadsCompleted("PhysicalReadRequestsCompleted", cc), transactionGetKeyRequests("GetKeyRequests", cc), transactionGetValueRequests("GetValueRequests", cc),
transactionGetRangeRequests("GetRangeRequests", cc), transactionWatchRequests("WatchRequests", cc), transactionGetAddressesForKeyRequests("GetAddressesForKeyRequests", cc),
transactionBytesRead("BytesRead", cc), transactionKeysRead("KeysRead", cc), transactionMetadataVersionReads("MetadataVersionReads", cc), transactionCommittedMutations("CommittedMutations", cc),
transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionSetMutations("SetMutations", cc), transactionClearMutations("ClearMutations", cc),
transactionAtomicMutations("AtomicMutations", cc), transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc),
transactionKeyServerLocationRequests("KeyServerLocationRequests", cc), transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc), transactionsTooOld("TooOld", cc),
transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc),
transactionsResourceConstrained("ResourceConstrained", cc), transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), latencies(1000), readLatencies(1000), commitLatencies(1000),
GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000),
internal(false) {}
@ -2290,7 +2323,7 @@ void getRangeFinished(Database cx, Reference<TransactionLogInfo> trLogInfo, doub
cx->transactionBytesRead += bytes;
cx->transactionKeysRead += result.size();
if( trLogInfo ) {
trLogInfo->addLog(FdbClientLogEvents::EventGetRange(startTime, cx->clientLocality.dcId(), now()-startTime, bytes, begin.getKey(), end.getKey()));
}
@ -2560,11 +2593,11 @@ Future<Standalone<RangeResultRef>> getRange( Database const& cx, Future<Version>
}
bool DatabaseContext::debugUseTags = false;
const std::vector<std::string> DatabaseContext::debugTransactionTagChoices = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t" };
const std::vector<std::string> DatabaseContext::debugTransactionTagChoices = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t" };
void debugAddTags(Transaction *tr) {
int numTags = deterministicRandom()->randomInt(0, CLIENT_KNOBS->MAX_TAGS_PER_TRANSACTION+1);
for(int i = 0; i < numTags; ++i) {
for(int i = 0; i < numTags; ++i) {
TransactionTag tag;
if(deterministicRandom()->random01() < 0.7) {
tag = TransactionTagRef(deterministicRandom()->randomChoice(DatabaseContext::debugTransactionTagChoices));
@ -2904,7 +2937,7 @@ void Transaction::set( const KeyRef& key, const ValueRef& value, bool addConflic
auto &t = req.transaction;
auto r = singleKeyRange( key, req.arena );
auto v = ValueRef( req.arena, value );
t.mutations.push_back( req.arena, MutationRef( MutationRef::SetValue, r.begin, v ) );
t.mutations.emplace_back(req.arena, MutationRef::SetValue, r.begin, v);
if( addConflictRange ) {
t.write_conflict_ranges.push_back( req.arena, r );
@ -2930,7 +2963,7 @@ void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationR
auto r = singleKeyRange( key, req.arena );
auto v = ValueRef( req.arena, operand );
t.mutations.push_back( req.arena, MutationRef( operationType, r.begin, v ) );
t.mutations.emplace_back(req.arena, operationType, r.begin, v);
if (addConflictRange && operationType != MutationRef::SetVersionstampedKey)
t.write_conflict_ranges.push_back( req.arena, r );
@ -2956,7 +2989,7 @@ void Transaction::clear( const KeyRangeRef& range, bool addConflictRange ) {
auto r = KeyRangeRef( req.arena, KeyRangeRef(begin, end) );
if (r.empty()) return;
t.mutations.push_back( req.arena, MutationRef( MutationRef::ClearRange, r.begin, r.end ) );
t.mutations.emplace_back(req.arena, MutationRef::ClearRange, r.begin, r.end);
if(addConflictRange)
t.write_conflict_ranges.push_back( req.arena, r );
@ -2974,10 +3007,11 @@ void Transaction::clear( const KeyRef& key, bool addConflictRange ) {
uint8_t* data = new ( req.arena ) uint8_t[ key.size()+1 ];
memcpy(data, key.begin(), key.size() );
data[key.size()] = 0;
t.mutations.push_back( req.arena, MutationRef( MutationRef::ClearRange, KeyRef(data,key.size()), KeyRef(data, key.size()+1)) );
t.mutations.emplace_back(req.arena, MutationRef::ClearRange, KeyRef(data, key.size()),
KeyRef(data, key.size() + 1));
if(addConflictRange)
t.write_conflict_ranges.push_back( req.arena, KeyRangeRef( KeyRef(data,key.size()), KeyRef(data, key.size()+1) ) );
t.write_conflict_ranges.emplace_back(req.arena, KeyRef(data, key.size()), KeyRef(data, key.size() + 1));
}
void Transaction::addWriteConflictRange( const KeyRangeRef& keys ) {
ASSERT( !keys.empty() );
@ -3483,7 +3517,8 @@ Future<Void> Transaction::commitMutations() {
bool isCheckingWrites = options.checkWritesEnabled && deterministicRandom()->random01() < 0.01;
for(int i=0; i<extraConflictRanges.size(); i++)
if (extraConflictRanges[i].isReady() && extraConflictRanges[i].get().first < extraConflictRanges[i].get().second )
tr.transaction.read_conflict_ranges.push_back( tr.arena, KeyRangeRef(extraConflictRanges[i].get().first, extraConflictRanges[i].get().second) );
tr.transaction.read_conflict_ranges.emplace_back(tr.arena, extraConflictRanges[i].get().first,
extraConflictRanges[i].get().second);
if( !options.causalWriteRisky && !intersects( tr.transaction.write_conflict_ranges, tr.transaction.read_conflict_ranges ).present() )
makeSelfConflicting();
@ -3632,7 +3667,7 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional<Stri
if (info.debugID.present()) {
TraceEvent(SevInfo, "TransactionBeingTraced")
.detail("DebugTransactionID", trLogInfo->identifier)
.detail("ServerTraceID", info.debugID.get().toString());
.detail("ServerTraceID", info.debugID.get().first());
}
break;
@ -3668,7 +3703,7 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional<Stri
if (trLogInfo && !trLogInfo->identifier.empty()) {
TraceEvent(SevInfo, "TransactionBeingTraced")
.detail("DebugTransactionID", trLogInfo->identifier)
.detail("ServerTraceID", info.debugID.get().toString());
.detail("ServerTraceID", info.debugID.get().first());
}
break;
@ -3858,11 +3893,11 @@ ACTOR Future<Void> readVersionBatcher( DatabaseContext *cx, FutureStream<Databas
}
}
ACTOR Future<Version> extractReadVersion(SpanID parentSpan, DatabaseContext* cx, TransactionPriority priority,
ACTOR Future<Version> extractReadVersion(Location location, SpanID spanContext, SpanID parent, DatabaseContext* cx, TransactionPriority priority,
Reference<TransactionLogInfo> trLogInfo, Future<GetReadVersionReply> f,
bool lockAware, double startTime, Promise<Optional<Value>> metadataVersion,
TagSet tags) {
// parentSpan here is only used to keep the parent alive until the request completes
state Span span(spanContext, location, { parent });
GetReadVersionReply rep = wait(f);
double latency = now() - startTime;
cx->GRVLatencies.addSample(latency);
@ -3934,7 +3969,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
case TransactionPriority::DEFAULT:
flags |= GetReadVersionRequest::PRIORITY_DEFAULT;
++cx->transactionDefaultReadVersions;
break;
break;
case TransactionPriority::BATCH:
flags |= GetReadVersionRequest::PRIORITY_BATCH;
++cx->transactionBatchReadVersions;
@ -3984,12 +4019,13 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
batcher.actor = readVersionBatcher( cx.getPtr(), batcher.stream.getFuture(), options.priority, flags );
}
Span span("NAPI:getReadVersion"_loc, info.spanID);
auto const req = DatabaseContext::VersionRequest(span.context, options.tags, info.debugID);
Location location = "NAPI:getReadVersion"_loc;
UID spanContext = deterministicRandom()->randomUniqueID();
auto const req = DatabaseContext::VersionRequest(spanContext, options.tags, info.debugID);
batcher.stream.send(req);
startTime = now();
readVersion = extractReadVersion(span.context, cx.getPtr(), options.priority, trLogInfo, req.reply.getFuture(),
options.lockAware, startTime, metadataVersion, options.tags);
readVersion = extractReadVersion(location, spanContext, info.spanID, cx.getPtr(), options.priority, trLogInfo,
req.reply.getFuture(), options.lockAware, startTime, metadataVersion, options.tags);
}
return readVersion;
}
@ -4169,7 +4205,7 @@ ACTOR Future< StorageMetrics > extractMetrics( Future<std::pair<Optional<Storage
return x.first.get();
}
ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getReadHotRanges(Database cx, KeyRange keys) {
ACTOR Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Database cx, KeyRange keys) {
state Span span("NAPI:GetReadHotRanges"_loc);
loop {
int64_t shardLimit = 100; // Shard limit here does not really matter since this function is currently only used
@ -4197,7 +4233,7 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getReadHotRanges(Database cx, K
}
wait(waitForAll(fReplies));
Standalone<VectorRef<KeyRangeRef>> results;
Standalone<VectorRef<ReadHotRangeWithMetrics>> results;
for (int i = 0; i < nLocs; i++)
results.append(results.arena(), fReplies[i].get().readHotRanges.begin(),
@ -4214,7 +4250,7 @@ ACTOR Future<Standalone<VectorRef<KeyRangeRef>>> getReadHotRanges(Database cx, K
}
}
}
ACTOR Future< std::pair<Optional<StorageMetrics>, int> > waitStorageMetrics(
Database cx,
KeyRange keys,
@ -4306,7 +4342,7 @@ ACTOR Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsLis
}
}
Future<Standalone<VectorRef<KeyRangeRef>>> Transaction::getReadHotRanges(KeyRange const& keys) {
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> Transaction::getReadHotRanges(KeyRange const& keys) {
return ::getReadHotRanges(cx, keys);
}

2
fdbclient/NativeAPI.actor.h
View File

@ -266,7 +266,7 @@ public:
// Pass a negative value for `shardLimit` to indicate no limit on the shard number.
Future< StorageMetrics > getStorageMetrics( KeyRange const& keys, int shardLimit );
Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated );
Future<Standalone<VectorRef<KeyRangeRef>>> getReadHotRanges(KeyRange const& keys);
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(KeyRange const& keys);
// If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key
void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true );

65
fdbclient/ReadYourWrites.actor.cpp
View File

@ -1045,6 +1045,9 @@ public:
try {
ryw->commitStarted = true;
if (ryw->options.specialKeySpaceChangeConfiguration)
wait(ryw->getDatabase()->specialKeySpace->commit(ryw));
Future<Void> ready = ryw->reading;
wait( ryw->resetPromise.getFuture() || ready );
@ -1160,7 +1163,8 @@ public:
ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx)
: cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0), creationTime(now()), commitStarted(false),
options(tr), deferredError(cx->deferredError), versionStampFuture(tr.getVersionstamp()) {
options(tr), deferredError(cx->deferredError), versionStampFuture(tr.getVersionstamp()),
specialKeySpaceWriteMap(std::make_pair(false, Optional<Value>()), specialKeys.end) {
std::copy(cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(),
std::back_inserter(persistentOptions));
applyPersistentOptions();
@ -1387,7 +1391,7 @@ Future< Standalone<VectorRef<const char*> >> ReadYourWritesTransaction::getAddre
return result;
}
Future<int64_t> ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
Future<int64_t> ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyRange& keys) {
if(checkUsedDuringCommit()) {
throw used_during_commit();
}
@ -1755,18 +1759,34 @@ void ReadYourWritesTransaction::atomicOp( const KeyRef& key, const ValueRef& ope
}
void ReadYourWritesTransaction::set( const KeyRef& key, const ValueRef& value ) {
if (key == LiteralStringRef("\xff\xff/reboot_worker")){
BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest() );
return;
}
if (key == LiteralStringRef("\xff\xff/reboot_and_check_worker")){
BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion()).reboot.send( RebootRequest(false, true) );
return;
}
if (key == metadataVersionKey) {
throw client_invalid_operation();
}
if (specialKeys.contains(key)) {
if (getDatabase()->apiVersionAtLeast(700)) {
return getDatabase()->specialKeySpace->set(this, key, value);
} else {
// These three special keys are deprecated in 7.0 and an alternative C API is added
// TODO : Rewrite related code using C api
if (key == LiteralStringRef("\xff\xff/reboot_worker")) {
BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
.reboot.send(RebootRequest());
return;
}
if (key == LiteralStringRef("\xff\xff/suspend_worker")){
BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
.reboot.send(RebootRequest(false, false, options.timeoutInSeconds));
return;
}
if (key == LiteralStringRef("\xff\xff/reboot_and_check_worker")) {
BinaryReader::fromStringRef<ClientWorkerInterface>(value, IncludeVersion())
.reboot.send(RebootRequest(false, true));
return;
}
}
}
bool addWriteConflict = !options.getAndResetWriteConflictDisabled();
if(checkUsedDuringCommit()) {
@ -1802,6 +1822,12 @@ void ReadYourWritesTransaction::clear( const KeyRangeRef& range ) {
throw used_during_commit();
}
if (specialKeys.contains(range)) {
if (getDatabase()->apiVersionAtLeast(700)) {
return getDatabase()->specialKeySpace->clear(this, range);
}
}
KeyRef maxKey = getMaxWriteKey();
if(range.begin > maxKey || range.end > maxKey)
throw key_outside_legal_range();
@ -1841,6 +1867,12 @@ void ReadYourWritesTransaction::clear( const KeyRef& key ) {
throw used_during_commit();
}
if (specialKeys.contains(key)) {
if (getDatabase()->apiVersionAtLeast(700)) {
return getDatabase()->specialKeySpace->clear(this, key);
}
}
if(key >= getMaxWriteKey())
throw key_outside_legal_range();
@ -2017,6 +2049,14 @@ void ReadYourWritesTransaction::setOptionImpl( FDBTransactionOptions::Option opt
case FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED:
validateOptionValue(value, false);
options.specialKeySpaceRelaxed = true;
break;
case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES:
validateOptionValue(value, false);
options.specialKeySpaceChangeConfiguration = true;
// By default, it allows to read system keys
// More options will be implicitly enabled if needed when doing set or clear
options.readSystemKeys = true;
break;
default:
break;
}
@ -2048,6 +2088,7 @@ void ReadYourWritesTransaction::operator=(ReadYourWritesTransaction&& r) noexcep
nativeReadRanges = std::move(r.nativeReadRanges);
nativeWriteRanges = std::move(r.nativeWriteRanges);
versionStampKeys = std::move(r.versionStampKeys);
specialKeySpaceWriteMap = std::move(r.specialKeySpaceWriteMap);
}
ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept
@ -2066,6 +2107,7 @@ ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&&
nativeReadRanges = std::move(r.nativeReadRanges);
nativeWriteRanges = std::move(r.nativeWriteRanges);
versionStampKeys = std::move(r.versionStampKeys);
specialKeySpaceWriteMap = std::move(r.specialKeySpaceWriteMap);
}
Future<Void> ReadYourWritesTransaction::onError(Error const& e) {
@ -2103,6 +2145,9 @@ void ReadYourWritesTransaction::resetRyow() {
versionStampKeys = VectorRef<KeyRef>();
nativeReadRanges = Standalone<VectorRef<KeyRangeRef>>();
nativeWriteRanges = Standalone<VectorRef<KeyRangeRef>>();
specialKeySpaceWriteMap =
KeyRangeMap<std::pair<bool, Optional<Value>>>(std::make_pair(false, Optional<Value>()), specialKeys.end);
specialKeySpaceErrorMsg.reset();
watchMap.clear();
reading = AndFuture();
approximateSize = 0;

13
fdbclient/ReadYourWrites.h
View File

@ -38,6 +38,7 @@ struct ReadYourWritesTransactionOptions {
bool debugRetryLogging : 1;
bool disableUsedDuringCommitProtection : 1;
bool specialKeySpaceRelaxed : 1;
bool specialKeySpaceChangeConfiguration : 1;
double timeoutInSeconds;
int maxRetries;
int snapshotRywEnabled;
@ -85,7 +86,7 @@ public:
}
[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
Future<int64_t> getEstimatedRangeSizeBytes( const KeyRangeRef& keys );
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys);
void addReadConflictRange( KeyRangeRef const& keys );
void makeSelfConflicting() { tr.makeSelfConflicting(); }
@ -148,6 +149,13 @@ public:
Standalone<RangeResultRef> getWriteConflictRangeIntersecting(KeyRangeRef kr);
bool specialKeySpaceRelaxed() const { return options.specialKeySpaceRelaxed; }
bool specialKeySpaceChangeConfiguration() const { return options.specialKeySpaceChangeConfiguration; }
KeyRangeMap<std::pair<bool, Optional<Value>>>& getSpecialKeySpaceWriteMap() { return specialKeySpaceWriteMap; }
bool readYourWritesDisabled() const { return options.readYourWritesDisabled; }
const Optional<std::string>& getSpecialKeySpaceErrorMsg() { return specialKeySpaceErrorMsg; }
void setSpecialKeySpaceErrorMsg(const std::string& msg) { specialKeySpaceErrorMsg = msg; }
Transaction& getTransaction() { return tr; }
private:
friend class RYWImpl;
@ -176,6 +184,9 @@ private:
Reference<TransactionDebugInfo> transactionDebugInfo;
KeyRangeMap<std::pair<bool, Optional<Value>>> specialKeySpaceWriteMap;
Optional<std::string> specialKeySpaceErrorMsg;
void resetTimeout();
void updateConflictMap( KeyRef const& key, WriteMap::iterator& it ); // pre: it.segmentContains(key)
void updateConflictMap( KeyRangeRef const& keys, WriteMap::iterator& it ); // pre: it.segmentContains(keys.begin), keys are already inside this->arena

78
fdbclient/RestoreWorkerInterface.actor.h
View File

@ -53,6 +53,7 @@ struct RestoreSendVersionedMutationsRequest;
struct RestoreSysInfo;
struct RestoreApplierInterface;
struct RestoreFinishRequest;
struct RestoreSamplesRequest;
// RestoreSysInfo includes information each (type of) restore roles should know.
// At this moment, it only include appliers. We keep the name for future extension.
@ -203,6 +204,31 @@ struct RestoreApplierInterface : RestoreRoleInterface {
std::string toString() { return nodeID.toString(); }
};
struct RestoreControllerInterface : RestoreRoleInterface {
constexpr static FileIdentifier file_identifier = 11642024;
RequestStream<RestoreSamplesRequest> samples;
bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); }
RestoreControllerInterface() {
role = RestoreRole::Controller;
nodeID = deterministicRandom()->randomUniqueID();
}
NetworkAddress address() const { return samples.getEndpoint().addresses.address; }
void initEndpoints() { samples.getEndpoint(TaskPriority::LoadBalancedEndpoint); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, *(RestoreRoleInterface*)this, samples);
}
std::string toString() { return nodeID.toString(); }
};
// RestoreAsset uniquely identifies the work unit done by restore roles;
// It is used to ensure exact-once processing on restore loader and applier;
// By combining all RestoreAssets across all verstion batches, restore should process all mutations in
@ -361,22 +387,25 @@ struct RestoreRecruitRoleReply : TimedRequest {
struct RestoreRecruitRoleRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 3136280;
RestoreControllerInterface ci;
RestoreRole role;
int nodeIndex; // Each role is a node
ReplyPromise<RestoreRecruitRoleReply> reply;
RestoreRecruitRoleRequest() : role(RestoreRole::Invalid) {}
explicit RestoreRecruitRoleRequest(RestoreRole role, int nodeIndex) : role(role), nodeIndex(nodeIndex) {}
explicit RestoreRecruitRoleRequest(RestoreControllerInterface ci, RestoreRole role, int nodeIndex)
: ci(ci), role(role), nodeIndex(nodeIndex) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, role, nodeIndex, reply);
serializer(ar, ci, role, nodeIndex, reply);
}
std::string printable() {
std::stringstream ss;
ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex;
ss << "RestoreRecruitRoleRequest Role:" << getRoleStr(role) << " NodeIndex:" << nodeIndex
<< " RestoreController:" << ci.id().toString();
return ss.str();
}
@ -410,26 +439,47 @@ struct RestoreSysInfoRequest : TimedRequest {
}
};
struct RestoreLoadFileReply : TimedRequest {
constexpr static FileIdentifier file_identifier = 523470;
struct RestoreSamplesRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 10751035;
UID id; // deduplicate data
int batchIndex;
SampledMutationsVec samples; // sampled mutations
LoadingParam param;
MutationsVec samples; // sampled mutations
bool isDuplicated; // true if loader thinks the request is a duplicated one
ReplyPromise<RestoreCommonReply> reply;
RestoreLoadFileReply() = default;
explicit RestoreLoadFileReply(LoadingParam param, MutationsVec samples, bool isDuplicated)
: param(param), samples(samples), isDuplicated(isDuplicated) {}
RestoreSamplesRequest() = default;
explicit RestoreSamplesRequest(UID id, int batchIndex, SampledMutationsVec samples)
: id(id), batchIndex(batchIndex), samples(samples) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, samples, isDuplicated);
serializer(ar, id, batchIndex, samples, reply);
}
std::string toString() {
std::stringstream ss;
ss << "LoadingParam:" << param.toString() << " samples.size:" << samples.size()
<< " isDuplicated:" << isDuplicated;
ss << "ID:" << id.toString() << " BatchIndex:" << batchIndex << " samples:" << samples.size();
return ss.str();
}
};
struct RestoreLoadFileReply : TimedRequest {
constexpr static FileIdentifier file_identifier = 523470;
LoadingParam param;
bool isDuplicated; // true if loader thinks the request is a duplicated one
RestoreLoadFileReply() = default;
explicit RestoreLoadFileReply(LoadingParam param, bool isDuplicated) : param(param), isDuplicated(isDuplicated) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, param, isDuplicated);
}
std::string toString() {
std::stringstream ss;
ss << "LoadingParam:" << param.toString() << " isDuplicated:" << isDuplicated;
return ss.str();
}
};

26
fdbclient/RunTransaction.actor.h
View File

@ -27,18 +27,21 @@
#elif !defined(FDBCLIENT_RUNTRANSACTION_ACTOR_H)
#define FDBCLIENT_RUNTRANSACTION_ACTOR_H
#include <utility>
#include "flow/flow.h"
#include "fdbclient/ReadYourWrites.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR template < class Function >
Future<decltype(fake<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
runRYWTransaction(Database cx, Function func) {
ACTOR template <class Function>
Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())> runRYWTransaction(
Database cx, Function func) {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
loop{
try {
// func should be idempodent; otherwise, retry will get undefined result
state decltype( fake<Function>()( Reference<ReadYourWritesTransaction>() ).getValue()) result = wait(func(tr));
state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result =
wait(func(tr));
wait(tr->commit());
return result;
}
@ -48,13 +51,14 @@ runRYWTransaction(Database cx, Function func) {
}
}
ACTOR template < class Function >
Future<decltype(fake<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
ACTOR template <class Function>
Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
runRYWTransactionFailIfLocked(Database cx, Function func) {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
loop{
try {
state decltype( fake<Function>()( Reference<ReadYourWritesTransaction>() ).getValue()) result = wait(func(tr));
state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result =
wait(func(tr));
wait(tr->commit());
return result;
}
@ -66,11 +70,11 @@ runRYWTransactionFailIfLocked(Database cx, Function func) {
}
}
ACTOR template < class Function >
Future<decltype(fake<Function>()(Reference<ReadYourWritesTransaction>()).getValue())>
runRYWTransactionNoRetry(Database cx, Function func) {
ACTOR template <class Function>
Future<decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue())> runRYWTransactionNoRetry(
Database cx, Function func) {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
state decltype(fake<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result = wait(func(tr));
state decltype(std::declval<Function>()(Reference<ReadYourWritesTransaction>()).getValue()) result = wait(func(tr));
wait(tr->commit());
return result;
}

8
fdbclient/Schemas.cpp
View File

@ -922,3 +922,11 @@ const KeyRef JSONSchemas::aggregateHealthSchema = LiteralStringRef(R"""(
"worst_log_queue": 156
}
)""");
const KeyRef JSONSchemas::managementApiErrorSchema = LiteralStringRef(R"""(
{
"retriable": false,
"command": "exclude",
"message": "The reason of the error"
}
)""");

1
fdbclient/Schemas.h
View File

@ -34,6 +34,7 @@ struct JSONSchemas {
static const KeyRef logHealthSchema;
static const KeyRef storageHealthSchema;
static const KeyRef aggregateHealthSchema;
static const KeyRef managementApiErrorSchema;
};
#endif /* FDBCLIENT_SCHEMAS_H */

616
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -20,6 +20,8 @@
#include "fdbclient/SpecialKeySpace.actor.h"
#include "flow/UnitTest.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/StatusClient.h"
#include "flow/actorcompiler.h" // This must be the last #include.
std::unordered_map<SpecialKeySpace::MODULE, KeyRange> SpecialKeySpace::moduleToBoundary = {
@ -31,16 +33,28 @@ std::unordered_map<SpecialKeySpace::MODULE, KeyRange> SpecialKeySpace::moduleToB
{ SpecialKeySpace::MODULE::CONNECTIONSTRING, singleKeyRange(LiteralStringRef("\xff\xff/connection_string")) },
{ SpecialKeySpace::MODULE::CLUSTERFILEPATH, singleKeyRange(LiteralStringRef("\xff\xff/cluster_file_path")) },
{ SpecialKeySpace::MODULE::METRICS,
KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) }
KeyRangeRef(LiteralStringRef("\xff\xff/metrics/"), LiteralStringRef("\xff\xff/metrics0")) },
{ SpecialKeySpace::MODULE::MANAGEMENT,
KeyRangeRef(LiteralStringRef("\xff\xff/management/"), LiteralStringRef("\xff\xff/management0")) },
{ SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) }
};
std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandToRange = {
{ "exclude", KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0"))
.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) },
{ "failed", KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0"))
.withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin) }
};
std::set<std::string> SpecialKeySpace::options = { "excluded/force", "failed/force" };
// This function will move the given KeySelector as far as possible to the standard form:
// orEqual == false && offset == 1 (Standard form)
// If the corresponding key is not in the underlying key range, it will move over the range
// The cache object is used to cache the first read result from the rpc call during the key resolution,
// then when we need to do key resolution or result filtering,
// we, instead of rpc call, read from this cache object have consistent results
ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeBaseImpl* skrImpl, ReadYourWritesTransaction* ryw,
ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeReadImpl* skrImpl, ReadYourWritesTransaction* ryw,
KeySelector* ks, Optional<Standalone<RangeResultRef>>* cache) {
ASSERT(!ks->orEqual); // should be removed before calling
ASSERT(ks->offset != 1); // never being called if KeySelector is already normalized
@ -91,7 +105,9 @@ ACTOR Future<Void> moveKeySelectorOverRangeActor(const SpecialKeyRangeBaseImpl*
ks->setKey(KeyRef(ks->arena(), result[ks->offset - 1].key));
ks->offset = 1;
} else {
ks->setKey(KeyRef(ks->arena(), keyAfter(result[result.size() - 1].key)));
ks->setKey(KeyRef(
ks->arena(),
keyAfter(result[result.size() - 1].key))); // TODO : the keyAfter will just return if key == \xff\xff
ks->offset -= result.size();
}
}
@ -115,9 +131,9 @@ ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks, ReadYourWrite
KeyRangeRef boundary, int* actualOffset,
Standalone<RangeResultRef>* result,
Optional<Standalone<RangeResultRef>>* cache) {
state RangeMap<Key, SpecialKeyRangeBaseImpl*, KeyRangeRef>::iterator iter =
ks->offset < 1 ? sks->getImpls().rangeContainingKeyBefore(ks->getKey())
: sks->getImpls().rangeContaining(ks->getKey());
state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter =
ks->offset < 1 ? sks->getReadImpls().rangeContainingKeyBefore(ks->getKey())
: sks->getReadImpls().rangeContaining(ks->getKey());
while ((ks->offset < 1 && iter->begin() > boundary.begin) || (ks->offset > 1 && iter->begin() < boundary.end)) {
if (iter->value() != nullptr) {
wait(moveKeySelectorOverRangeActor(iter->value(), ryw, ks, cache));
@ -141,6 +157,30 @@ ACTOR Future<Void> normalizeKeySelectorActor(SpecialKeySpace* sks, ReadYourWrite
return Void();
}
SpecialKeySpace::SpecialKeySpace(KeyRef spaceStartKey, KeyRef spaceEndKey, bool testOnly)
: range(KeyRangeRef(spaceStartKey, spaceEndKey)), readImpls(nullptr, spaceEndKey), writeImpls(nullptr, spaceEndKey),
modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey) {
// Default begin of KeyRangeMap is Key(), insert the range to update start key
readImpls.insert(range, nullptr);
writeImpls.insert(range, nullptr);
if (!testOnly) modulesBoundaryInit(); // testOnly is used in the correctness workload
}
void SpecialKeySpace::modulesBoundaryInit() {
for (const auto& pair : moduleToBoundary) {
ASSERT(range.contains(pair.second));
// Make sure the module is not overlapping with any registered read modules
// Note: same like ranges, one module's end cannot be another module's start, relax the condition if needed
ASSERT(modules.rangeContaining(pair.second.begin) == modules.rangeContaining(pair.second.end) &&
modules[pair.second.begin] == SpecialKeySpace::MODULE::UNKNOWN);
modules.insert(pair.second, pair.first);
// Note: Due to underlying implementation, the insertion here is important to make cross_module_read being
// handled correctly
readImpls.insert(pair.second, nullptr);
writeImpls.insert(pair.second, nullptr);
}
}
ACTOR Future<Standalone<RangeResultRef>> SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks,
ReadYourWritesTransaction* ryw,
KeySelector begin, KeySelector end,
@ -164,7 +204,7 @@ ACTOR Future<Standalone<RangeResultRef>> SpecialKeySpace::getRangeAggregationAct
// KeySelector, GetRangeLimits and reverse are all handled here
state Standalone<RangeResultRef> result;
state Standalone<RangeResultRef> pairs;
state RangeMap<Key, SpecialKeyRangeBaseImpl*, KeyRangeRef>::iterator iter;
state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::iterator iter;
state int actualBeginOffset;
state int actualEndOffset;
state KeyRangeRef moduleBoundary;
@ -203,8 +243,8 @@ ACTOR Future<Standalone<RangeResultRef>> SpecialKeySpace::getRangeAggregationAct
TEST(true);
return result;
}
state RangeMap<Key, SpecialKeyRangeBaseImpl*, KeyRangeRef>::Ranges ranges =
sks->impls.intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey()));
state RangeMap<Key, SpecialKeyRangeReadImpl*, KeyRangeRef>::Ranges ranges =
sks->getReadImpls().intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey()));
// TODO : workaround to write this two together to make the code compact
// The issue here is boost::iterator_range<> doest not provide rbegin(), rend()
iter = reverse ? ranges.end() : ranges.begin();
@ -226,6 +266,7 @@ ACTOR Future<Standalone<RangeResultRef>> SpecialKeySpace::getRangeAggregationAct
result.arena().dependsOn(pairs.arena());
// limits handler
for (int i = pairs.size() - 1; i >= 0; --i) {
ASSERT(iter->range().contains(pairs[i].key));
result.push_back(result.arena(), pairs[i]);
// Note : behavior here is even the last k-v pair makes total bytes larger than specified, it's still
// returned. In other words, the total size of the returned value (less the last entry) will be less
@ -255,6 +296,7 @@ ACTOR Future<Standalone<RangeResultRef>> SpecialKeySpace::getRangeAggregationAct
result.arena().dependsOn(pairs.arena());
// limits handler
for (int i = 0; i < pairs.size(); ++i) {
ASSERT(iter->range().contains(pairs[i].key));
result.push_back(result.arena(), pairs[i]);
// Note : behavior here is even the last k-v pair makes total bytes larger than specified, it's still
// returned. In other words, the total size of the returned value (less the last entry) will be less
@ -309,7 +351,112 @@ Future<Optional<Value>> SpecialKeySpace::get(ReadYourWritesTransaction* ryw, con
return getActor(this, ryw, key);
}
ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {}
void SpecialKeySpace::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
if (!ryw->specialKeySpaceChangeConfiguration()) throw special_keys_write_disabled();
auto impl = writeImpls[key];
if (impl == nullptr) {
TraceEvent(SevDebug, "SpecialKeySpaceNoWriteModuleFound")
.detail("Key", key.toString())
.detail("Value", value.toString());
throw special_keys_no_write_module_found();
}
return impl->set(ryw, key, value);
}
void SpecialKeySpace::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
if (!ryw->specialKeySpaceChangeConfiguration()) throw special_keys_write_disabled();
if (range.empty()) return;
auto begin = writeImpls[range.begin];
auto end = writeImpls.rangeContainingKeyBefore(range.end)->value();
if (begin != end) {
TraceEvent(SevDebug, "SpecialKeySpaceCrossModuleClear").detail("Range", range.toString());
throw special_keys_cross_module_clear(); // ban cross module clear
} else if (begin == nullptr) {
TraceEvent(SevDebug, "SpecialKeySpaceNoWriteModuleFound").detail("Range", range.toString());
throw special_keys_no_write_module_found();
}
return begin->clear(ryw, range);
}
void SpecialKeySpace::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
if (!ryw->specialKeySpaceChangeConfiguration()) throw special_keys_write_disabled();
auto impl = writeImpls[key];
if (impl == nullptr) throw special_keys_no_write_module_found();
return impl->clear(ryw, key);
}
void SpecialKeySpace::registerKeyRange(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type,
const KeyRangeRef& kr, SpecialKeyRangeReadImpl* impl) {
// module boundary check
if (module == SpecialKeySpace::MODULE::TESTONLY)
ASSERT(normalKeys.contains(kr));
else
ASSERT(moduleToBoundary.at(module).contains(kr));
// make sure the registered range is not overlapping with existing ones
// Note: kr.end should not be the same as another range's begin, although it should work even they are the same
for (auto iter = readImpls.rangeContaining(kr.begin); true; ++iter) {
ASSERT(iter->value() == nullptr);
if (iter == readImpls.rangeContaining(kr.end))
break; // Note: relax the condition that the end can be another range's start, if needed
}
readImpls.insert(kr, impl);
// if rw, it means the module can do both read and write
if (type == SpecialKeySpace::IMPLTYPE::READWRITE) {
// since write impls are always subset of read impls,
// no need to check overlapped registration
auto rwImpl = dynamic_cast<SpecialKeyRangeRWImpl*>(impl);
ASSERT(rwImpl);
writeImpls.insert(kr, rwImpl);
}
}
Key SpecialKeySpace::decode(const KeyRef& key) {
auto impl = writeImpls[key];
ASSERT(impl != nullptr);
return impl->decode(key);
}
KeyRange SpecialKeySpace::decode(const KeyRangeRef& kr) {
// Only allow to decode key range in the same underlying impl range
auto begin = writeImpls.rangeContaining(kr.begin);
ASSERT(begin->value() != nullptr);
auto end = writeImpls.rangeContainingKeyBefore(kr.end);
ASSERT(begin == end);
return KeyRangeRef(begin->value()->decode(kr.begin), begin->value()->decode(kr.end));
}
ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw) {
state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys);
state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
state std::set<SpecialKeyRangeRWImpl*> writeModulePtrs;
while (iter != ranges.end()) {
std::pair<bool, Optional<Value>> entry = iter->value();
if (entry.first) {
auto modulePtr = sks->getRWImpls().rangeContaining(iter->begin())->value();
writeModulePtrs.insert(modulePtr);
}
++iter;
}
state std::set<SpecialKeyRangeRWImpl*>::const_iterator it;
for (it = writeModulePtrs.begin(); it != writeModulePtrs.end(); ++it) {
Optional<std::string> msg = wait((*it)->commit(ryw));
if (msg.present()) {
ryw->setSpecialKeySpaceErrorMsg(msg.get());
TraceEvent(SevDebug, "SpecialKeySpaceManagemetnAPIError")
.detail("Reason", msg.get())
.detail("Range", (*it)->getKeyRange().toString());
throw special_keys_api_failure();
}
}
return Void();
}
Future<Void> SpecialKeySpace::commit(ReadYourWritesTransaction* ryw) {
return commitActor(this, ryw);
}
ReadConflictRangeImpl::ReadConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
ACTOR static Future<Standalone<RangeResultRef>> getReadConflictRangeImpl(ReadYourWritesTransaction* ryw, KeyRange kr) {
wait(ryw->pendingReads());
@ -321,14 +468,14 @@ Future<Standalone<RangeResultRef>> ReadConflictRangeImpl::getRange(ReadYourWrite
return getReadConflictRangeImpl(ryw, kr);
}
WriteConflictRangeImpl::WriteConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {}
WriteConflictRangeImpl::WriteConflictRangeImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
Future<Standalone<RangeResultRef>> WriteConflictRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
return ryw->getWriteConflictRangeIntersecting(kr);
}
ConflictingKeysImpl::ConflictingKeysImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {}
ConflictingKeysImpl::ConflictingKeysImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
Future<Standalone<RangeResultRef>> ConflictingKeysImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
Standalone<RangeResultRef> result;
@ -338,13 +485,10 @@ Future<Standalone<RangeResultRef>> ConflictingKeysImpl::getRange(ReadYourWritesT
if (beginIter->begin() != kr.begin) ++beginIter;
auto endIter = krMapPtr->rangeContaining(kr.end);
for (auto it = beginIter; it != endIter; ++it) {
// it->begin() is stored in the CoalescedKeyRangeMap in TransactionInfo
// it->value() is always constants in SystemData.cpp
// Thus, push_back() can be used
result.push_back(result.arena(), KeyValueRef(it->begin(), it->value()));
result.push_back_deep(result.arena(), KeyValueRef(it->begin(), it->value()));
}
if (endIter->begin() != kr.end)
result.push_back(result.arena(), KeyValueRef(endIter->begin(), endIter->value()));
result.push_back_deep(result.arena(), KeyValueRef(endIter->begin(), endIter->value()));
}
return result;
}
@ -377,3 +521,441 @@ DDStatsRangeImpl::DDStatsRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr
Future<Standalone<RangeResultRef>> DDStatsRangeImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const {
return ddMetricsGetRangeActor(ryw, kr);
}
Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option) {
Key prefix = LiteralStringRef("options/").withPrefix(moduleToBoundary[MODULE::MANAGEMENT].begin);
auto pair = command + "/" + option;
ASSERT(options.find(pair) != options.end());
return prefix.withSuffix(pair);
}
ManagementCommandsOptionsImpl::ManagementCommandsOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<Standalone<RangeResultRef>> ManagementCommandsOptionsImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
Standalone<RangeResultRef> result;
// Since we only have limit number of options, a brute force loop here is enough
for (const auto& option : SpecialKeySpace::getManagementApiOptionsSet()) {
auto key = getKeyRange().begin.withSuffix(option);
// ignore all invalid keys
auto r = ryw->getSpecialKeySpaceWriteMap()[key];
if (kr.contains(key) && r.first && r.second.present()) {
result.push_back(result.arena(), KeyValueRef(key, ValueRef()));
result.arena().dependsOn(key.arena());
}
}
return result;
}
void ManagementCommandsOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
std::string option = key.removePrefix(getKeyRange().begin).toString();
// ignore all invalid keys
if (SpecialKeySpace::getManagementApiOptionsSet().find(option) !=
SpecialKeySpace::getManagementApiOptionsSet().end()) {
TraceEvent(SevDebug, "ManagementApiOption").detail("Option", option).detail("Key", key);
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>(value)));
}
}
void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
ryw->getSpecialKeySpaceWriteMap().rawErase(range);
}
void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
std::string option = key.removePrefix(getKeyRange().begin).toString();
// ignore all invalid keys
if (SpecialKeySpace::getManagementApiOptionsSet().find(option) !=
SpecialKeySpace::getManagementApiOptionsSet().end()) {
ryw->getSpecialKeySpaceWriteMap().rawErase(singleKeyRange(key));
}
}
Key ManagementCommandsOptionsImpl::decode(const KeyRef& key) const {
// Should never be used
ASSERT(false);
return key;
}
Key ManagementCommandsOptionsImpl::encode(const KeyRef& key) const {
// Should never be used
ASSERT(false);
return key;
}
Future<Optional<std::string>> ManagementCommandsOptionsImpl::commit(ReadYourWritesTransaction* ryw) {
// Nothing to do, keys should be used by other impls' commit callback
return Optional<std::string>();
}
// read from rwModule
ACTOR Future<Standalone<RangeResultRef>> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw,
const SpecialKeyRangeRWImpl* impl, KeyRangeRef kr) {
state KeyRangeRef range = impl->getKeyRange();
Standalone<RangeResultRef> resultWithoutPrefix =
wait(ryw->getRange(ryw->getDatabase()->specialKeySpace->decode(kr), CLIENT_KNOBS->TOO_MANY));
ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY);
Standalone<RangeResultRef> result;
if (ryw->readYourWritesDisabled()) {
for (const KeyValueRef& kv : resultWithoutPrefix)
result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value));
} else {
RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
int index = 0;
while (iter != ranges.end()) {
// add all previous entries into result
Key rk = impl->encode(resultWithoutPrefix[index].key);
while (index < resultWithoutPrefix.size() && rk < iter->begin()) {
result.push_back_deep(result.arena(), KeyValueRef(rk, resultWithoutPrefix[index].value));
++index;
}
std::pair<bool, Optional<Value>> entry = iter->value();
if (entry.first) {
// add the writen entries if exists
if (entry.second.present()) {
result.push_back_deep(result.arena(), KeyValueRef(iter->begin(), entry.second.get()));
}
// move index to skip all entries in the iter->range
while (index < resultWithoutPrefix.size() &&
iter->range().contains(impl->encode(resultWithoutPrefix[index].key)))
++index;
}
++iter;
}
// add all remaining entries into result
while (index < resultWithoutPrefix.size()) {
const KeyValueRef& kv = resultWithoutPrefix[index];
result.push_back_deep(result.arena(), KeyValueRef(impl->encode(kv.key), kv.value));
++index;
}
}
return result;
}
ExcludeServersRangeImpl::ExcludeServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<Standalone<RangeResultRef>> ExcludeServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
return rwModuleGetRangeActor(ryw, this, kr);
}
void ExcludeServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>(value)));
}
void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>()));
}
void ExcludeServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional<Value>()));
}
Key ExcludeServersRangeImpl::decode(const KeyRef& key) const {
return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
.withPrefix(LiteralStringRef("\xff/conf/"));
}
Key ExcludeServersRangeImpl::encode(const KeyRef& key) const {
return key.removePrefix(LiteralStringRef("\xff/conf/"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
}
bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, bool failed, std::vector<AddressExclusion>& addresses,
std::set<AddressExclusion>& exclusions, Optional<std::string>& msg) {
KeyRangeRef range = failed ? SpecialKeySpace::getManamentApiCommandRange("failed")
: SpecialKeySpace::getManamentApiCommandRange("exclude");
auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
auto iter = ranges.begin();
while (iter != ranges.end()) {
auto entry = iter->value();
// only check for exclude(set) operation, include(clear) are not checked
TraceEvent(SevInfo, "ParseNetworkAddress")
.detail("Valid", entry.first)
.detail("Set", entry.second.present())
.detail("Key", iter->begin().toString());
if (entry.first && entry.second.present()) {
Key address = iter->begin().removePrefix(range.begin);
auto a = AddressExclusion::parse(address);
if (!a.isValid()) {
std::string error = "ERROR: \'" + address.toString() + "\' is not a valid network endpoint address\n";
if (address.toString().find(":tls") != std::string::npos)
error += " Do not include the `:tls' suffix when naming a process\n";
msg = ManagementAPIError::toJsonString(
false, entry.second.present() ? (failed ? "exclude failed" : "exclude") : "include", error);
return false;
}
addresses.push_back(a);
exclusions.insert(a);
}
++iter;
}
return true;
}
ACTOR Future<bool> checkExclusion(Database db, std::vector<AddressExclusion>* addresses,
std::set<AddressExclusion>* exclusions, bool markFailed, Optional<std::string>* msg) {
if (markFailed) {
state bool safe;
try {
bool _safe = wait(checkSafeExclusions(db, *addresses));
safe = _safe;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
TraceEvent("CheckSafeExclusionsError").error(e);
safe = false;
}
if (!safe) {
std::string temp = "ERROR: It is unsafe to exclude the specified servers at this time.\n"
"Please check that this exclusion does not bring down an entire storage team.\n"
"Please also ensure that the exclusion will keep a majority of coordinators alive.\n"
"You may add more storage processes or coordinators to make the operation safe.\n"
"Call set(\"0xff0xff/management/failed/<ADDRESS...>\", ...) to exclude without "
"performing safety checks.\n";
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
return false;
}
}
StatusObject status = wait(StatusClient::statusFetcher(db));
state std::string errorString =
"ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n"
"Please try the exclude again in 30 seconds.\n"
"Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without checking free "
"space.\n";
StatusObjectReader statusObj(status);
StatusObjectReader statusObjCluster;
if (!statusObj.get("cluster", statusObjCluster)) {
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
StatusObjectReader processesMap;
if (!statusObjCluster.get("processes", processesMap)) {
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
state int ssTotalCount = 0;
state int ssExcludedCount = 0;
state double worstFreeSpaceRatio = 1.0;
try {
for (auto proc : processesMap.obj()) {
bool storageServer = false;
StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
for (StatusObjectReader role : rolesArray) {
if (role["role"].get_str() == "storage") {
storageServer = true;
break;
}
}
// Skip non-storage servers in free space calculation
if (!storageServer) continue;
StatusObjectReader process(proc.second);
std::string addrStr;
if (!process.get("address", addrStr)) {
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
NetworkAddress addr = NetworkAddress::parse(addrStr);
bool excluded =
(process.has("excluded") && process.last().get_bool()) || addressExcluded(*exclusions, addr);
ssTotalCount++;
if (excluded) ssExcludedCount++;
if (!excluded) {
StatusObjectReader disk;
if (!process.get("disk", disk)) {
*msg =
ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
int64_t total_bytes;
if (!disk.get("total_bytes", total_bytes)) {
*msg =
ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
int64_t free_bytes;
if (!disk.get("free_bytes", free_bytes)) {
*msg =
ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
worstFreeSpaceRatio = std::min(worstFreeSpaceRatio, double(free_bytes) / total_bytes);
}
}
} catch (...) // std::exception
{
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString);
return false;
}
if (ssExcludedCount == ssTotalCount ||
(1 - worstFreeSpaceRatio) * ssTotalCount / (ssTotalCount - ssExcludedCount) > 0.9) {
std::string temp = "ERROR: This exclude may cause the total free space in the cluster to drop below 10%.\n"
"Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without "
"checking free space.\n";
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
return false;
}
return true;
}
void includeServers(ReadYourWritesTransaction* ryw) {
ryw->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
ryw->setOption(FDBTransactionOptions::LOCK_AWARE);
ryw->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
// includeServers might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
ryw->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
std::string versionKey = deterministicRandom()->randomUniqueID().toString();
// for exluded servers
auto ranges =
ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("exclude"));
auto iter = ranges.begin();
Transaction& tr = ryw->getTransaction();
while (iter != ranges.end()) {
auto entry = iter->value();
if (entry.first && !entry.second.present()) {
tr.addReadConflictRange(singleKeyRange(excludedServersVersionKey));
tr.set(excludedServersVersionKey, versionKey);
tr.clear(ryw->getDatabase()->specialKeySpace->decode(iter->range()));
}
++iter;
}
// for failed servers
ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(SpecialKeySpace::getManamentApiCommandRange("failed"));
iter = ranges.begin();
while (iter != ranges.end()) {
auto entry = iter->value();
if (entry.first && !entry.second.present()) {
tr.addReadConflictRange(singleKeyRange(failedServersVersionKey));
tr.set(failedServersVersionKey, versionKey);
tr.clear(ryw->getDatabase()->specialKeySpace->decode(iter->range()));
}
++iter;
}
}
ACTOR Future<Optional<std::string>> excludeCommitActor(ReadYourWritesTransaction* ryw, bool failed) {
// parse network addresses
state Optional<std::string> result;
state std::vector<AddressExclusion> addresses;
state std::set<AddressExclusion> exclusions;
if (!parseNetWorkAddrFromKeys(ryw, failed, addresses, exclusions, result)) return result;
// If force option is not set, we need to do safety check
auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
failed ? "failed" : "excluded", "force")];
// only do safety check when we have servers to be excluded and the force option key is not set
if (addresses.size() && !(force.first && force.second.present())) {
bool safe = wait(checkExclusion(ryw->getDatabase(), &addresses, &exclusions, failed, &result));
if (!safe) return result;
}
excludeServers(ryw->getTransaction(), addresses, failed);
includeServers(ryw);
return result;
}
Future<Optional<std::string>> ExcludeServersRangeImpl::commit(ReadYourWritesTransaction* ryw) {
return excludeCommitActor(ryw, false);
}
FailedServersRangeImpl::FailedServersRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<Standalone<RangeResultRef>> FailedServersRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
return rwModuleGetRangeActor(ryw, this, kr);
}
void FailedServersRangeImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>(value)));
}
void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>()));
}
void FailedServersRangeImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional<Value>()));
}
Key FailedServersRangeImpl::decode(const KeyRef& key) const {
return key.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
.withPrefix(LiteralStringRef("\xff/conf/"));
}
Key FailedServersRangeImpl::encode(const KeyRef& key) const {
return key.removePrefix(LiteralStringRef("\xff/conf/"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
}
Future<Optional<std::string>> FailedServersRangeImpl::commit(ReadYourWritesTransaction* ryw) {
return excludeCommitActor(ryw, true);
}
ACTOR Future<Standalone<RangeResultRef>> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix,
KeyRangeRef kr) {
state Standalone<RangeResultRef> result;
state Transaction& tr = ryw->getTransaction();
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
state std::vector<AddressExclusion> excl = wait((getExcludedServers(&tr)));
state std::set<AddressExclusion> exclusions(excl.begin(), excl.end());
state std::set<NetworkAddress> inProgressExclusion;
// Just getting a consistent read version proves that a set of tlogs satisfying the exclusions has completed
// recovery Check that there aren't any storage servers with addresses violating the exclusions
state Standalone<RangeResultRef> serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
for (auto& s : serverList) {
auto addresses = decodeServerListValue(s.value).getKeyValues.getEndpoint().addresses;
if (addressExcluded(exclusions, addresses.address)) {
inProgressExclusion.insert(addresses.address);
}
if (addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get())) {
inProgressExclusion.insert(addresses.secondaryAddress.get());
}
}
Optional<Standalone<StringRef>> value = wait(tr.get(logsKey));
ASSERT(value.present());
auto logs = decodeLogsValue(value.get());
for (auto const& log : logs.first) {
if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
inProgressExclusion.insert(log.second);
}
}
for (auto const& log : logs.second) {
if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) {
inProgressExclusion.insert(log.second);
}
}
for (auto const& address : inProgressExclusion) {
Key addrKey = prefix.withSuffix(address.toString());
if (kr.contains(addrKey)) {
result.push_back(result.arena(), KeyValueRef(addrKey, ValueRef()));
result.arena().dependsOn(addrKey.arena());
}
}
return result;
}
ExclusionInProgressRangeImpl::ExclusionInProgressRangeImpl(KeyRangeRef kr) : SpecialKeyRangeAsyncImpl(kr) {}
Future<Standalone<RangeResultRef>> ExclusionInProgressRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr) const {
return ExclusionInProgressActor(ryw, getKeyRange().begin, kr);
}

179
fdbclient/SpecialKeySpace.actor.h
View File

@ -33,27 +33,58 @@
#include "fdbclient/ReadYourWrites.h"
#include "flow/actorcompiler.h" // This must be the last #include.
class SpecialKeyRangeBaseImpl {
class SpecialKeyRangeReadImpl {
public:
// Each derived class only needs to implement this simple version of getRange
virtual Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const = 0;
explicit SpecialKeyRangeBaseImpl(KeyRangeRef kr) : range(kr) {}
explicit SpecialKeyRangeReadImpl(KeyRangeRef kr) : range(kr) {}
KeyRangeRef getKeyRange() const { return range; }
// true if the getRange call can emit more than one rpc calls,
// we cache the results to keep consistency in the same getrange lifetime
// TODO : give this function a more descriptive name
virtual bool isAsync() const { return false; }
virtual ~SpecialKeyRangeBaseImpl() {}
virtual ~SpecialKeyRangeReadImpl() {}
protected:
KeyRange range; // underlying key range for this function
};
class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeBaseImpl {
class ManagementAPIError {
public:
explicit SpecialKeyRangeAsyncImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {}
static std::string toJsonString(bool retriable, const std::string& command, const std::string& msg) {
json_spirit::mObject errorObj;
errorObj["retriable"] = retriable;
errorObj["command"] = command;
errorObj["message"] = msg;
return json_spirit::write_string(json_spirit::mValue(errorObj), json_spirit::Output_options::raw_utf8);
}
private:
ManagementAPIError(){};
};
class SpecialKeyRangeRWImpl : public SpecialKeyRangeReadImpl {
public:
virtual void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) = 0;
virtual void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) = 0;
virtual Future<Optional<std::string>> commit(
ReadYourWritesTransaction* ryw) = 0; // all delayed async operations of writes in special-key-space
// Given the special key to write, return the real key that needs to be modified
virtual Key decode(const KeyRef& key) const = 0;
// Given the read key, return the corresponding special key
virtual Key encode(const KeyRef& key) const = 0;
explicit SpecialKeyRangeRWImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
virtual ~SpecialKeyRangeRWImpl() {}
};
class SpecialKeyRangeAsyncImpl : public SpecialKeyRangeReadImpl {
public:
explicit SpecialKeyRangeAsyncImpl(KeyRangeRef kr) : SpecialKeyRangeReadImpl(kr) {}
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const = 0;
@ -65,7 +96,7 @@ public:
bool isAsync() const override { return true; }
ACTOR static Future<Standalone<RangeResultRef>> getRangeAsyncActor(const SpecialKeyRangeBaseImpl* skrAyncImpl,
ACTOR static Future<Standalone<RangeResultRef>> getRangeAsyncActor(const SpecialKeyRangeReadImpl* skrAyncImpl,
ReadYourWritesTransaction* ryw, KeyRangeRef kr,
Optional<Standalone<RangeResultRef>>* cache) {
ASSERT(skrAyncImpl->getKeyRange().contains(kr));
@ -95,6 +126,8 @@ public:
enum class MODULE {
CLUSTERFILEPATH,
CONNECTIONSTRING,
ERRORMSG, // A single key space contains a json string which describes the last error in special-key-space
MANAGEMENT, // Management-API
METRICS, // data-distribution metrics
TESTONLY, // only used by correctness tests
TRANSACTION, // transaction related info, conflicting keys, read/write conflict range
@ -103,67 +136,69 @@ public:
WORKERINTERFACE,
};
enum class IMPLTYPE {
READONLY, // The underlying special key range can only be called with get and getRange
READWRITE // The underlying special key range can be called with get, getRange, set, clear
};
SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = normalKeys.end, bool testOnly = true);
Future<Optional<Value>> get(ReadYourWritesTransaction* ryw, const Key& key);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeySelector begin, KeySelector end,
GetRangeLimits limits, bool reverse = false);
SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = normalKeys.end, bool testOnly = true)
: range(KeyRangeRef(spaceStartKey, spaceEndKey)), impls(nullptr, spaceEndKey),
modules(testOnly ? SpecialKeySpace::MODULE::TESTONLY : SpecialKeySpace::MODULE::UNKNOWN, spaceEndKey) {
// Default begin of KeyRangeMap is Key(), insert the range to update start key if needed
impls.insert(range, nullptr);
if (!testOnly) modulesBoundaryInit(); // testOnly is used in the correctness workload
}
// Initialize module boundaries, used to handle cross_module_read
void modulesBoundaryInit() {
for (const auto& pair : moduleToBoundary) {
ASSERT(range.contains(pair.second));
// Make sure the module is not overlapping with any registered modules
// Note: same like ranges, one module's end cannot be another module's start, relax the condition if needed
ASSERT(modules.rangeContaining(pair.second.begin) == modules.rangeContaining(pair.second.end) &&
modules[pair.second.begin] == SpecialKeySpace::MODULE::UNKNOWN);
modules.insert(pair.second, pair.first);
impls.insert(pair.second, nullptr); // Note: Due to underlying implementation, the insertion here is
// important to make cross_module_read being handled correctly
}
}
void registerKeyRange(SpecialKeySpace::MODULE module, const KeyRangeRef& kr, SpecialKeyRangeBaseImpl* impl) {
// module boundary check
if (module == SpecialKeySpace::MODULE::TESTONLY)
ASSERT(normalKeys.contains(kr));
else
ASSERT(moduleToBoundary.at(module).contains(kr));
// make sure the registered range is not overlapping with existing ones
// Note: kr.end should not be the same as another range's begin, although it should work even they are the same
for (auto iter = impls.rangeContaining(kr.begin); true; ++iter) {
ASSERT(iter->value() == nullptr);
if (iter == impls.rangeContaining(kr.end))
break; // relax the condition that the end can be another range's start, if needed
}
impls.insert(kr, impl);
}
void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value);
KeyRangeMap<SpecialKeyRangeBaseImpl*>& getImpls() { return impls; }
void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range);
void clear(ReadYourWritesTransaction* ryw, const KeyRef& key);
Future<Void> commit(ReadYourWritesTransaction* ryw);
void registerKeyRange(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type, const KeyRangeRef& kr,
SpecialKeyRangeReadImpl* impl);
Key decode(const KeyRef& key);
KeyRange decode(const KeyRangeRef& kr);
KeyRangeMap<SpecialKeyRangeReadImpl*>& getReadImpls() { return readImpls; }
KeyRangeMap<SpecialKeyRangeRWImpl*>& getRWImpls() { return writeImpls; }
KeyRangeMap<SpecialKeySpace::MODULE>& getModules() { return modules; }
KeyRangeRef getKeyRange() const { return range; }
static KeyRangeRef getModuleRange(SpecialKeySpace::MODULE module) { return moduleToBoundary.at(module); }
static KeyRangeRef getManamentApiCommandRange(const std::string& command) {
return managementApiCommandToRange.at(command);
}
static KeyRef getManagementApiCommandPrefix(const std::string& command) {
return managementApiCommandToRange.at(command).begin;
}
static Key getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option);
static const std::set<std::string>& getManagementApiOptionsSet() { return options; }
private:
ACTOR static Future<Optional<Value>> getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key);
ACTOR static Future<Standalone<RangeResultRef>> checkRYWValid(SpecialKeySpace* sks,
ReadYourWritesTransaction* ryw, KeySelector begin,
KeySelector end, GetRangeLimits limits,
bool reverse);
ACTOR static Future<Standalone<RangeResultRef>> checkRYWValid(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw,
KeySelector begin, KeySelector end,
GetRangeLimits limits, bool reverse);
ACTOR static Future<Standalone<RangeResultRef>> getRangeAggregationActor(SpecialKeySpace* sks,
ReadYourWritesTransaction* ryw,
KeySelector begin, KeySelector end,
GetRangeLimits limits, bool reverse);
KeyRange range;
KeyRangeMap<SpecialKeyRangeBaseImpl*> impls;
KeyRangeMap<SpecialKeyRangeReadImpl*> readImpls;
KeyRangeMap<SpecialKeySpace::MODULE> modules;
KeyRangeMap<SpecialKeyRangeRWImpl*> writeImpls;
KeyRange range; // key space range, (\xff\xff, \xff\xff\xff) in prod and (, \xff) in test
static std::unordered_map<SpecialKeySpace::MODULE, KeyRange> moduleToBoundary;
static std::unordered_map<std::string, KeyRange>
managementApiCommandToRange; // management command to its special keys' range
static std::set<std::string> options; // "<command>/<option>"
// Initialize module boundaries, used to handle cross_module_read
void modulesBoundaryInit();
};
// Use special key prefix "\xff\xff/transaction/conflicting_keys/<some_key>",
@ -172,19 +207,19 @@ private:
// prefix/<key1> : '1' - any keys equal or larger than this key are (probably) conflicting keys
// prefix/<key2> : '0' - any keys equal or larger than this key are (definitely) not conflicting keys
// Currently, the conflicting keyranges returned are original read_conflict_ranges or union of them.
class ConflictingKeysImpl : public SpecialKeyRangeBaseImpl {
class ConflictingKeysImpl : public SpecialKeyRangeReadImpl {
public:
explicit ConflictingKeysImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
};
class ReadConflictRangeImpl : public SpecialKeyRangeBaseImpl {
class ReadConflictRangeImpl : public SpecialKeyRangeReadImpl {
public:
explicit ReadConflictRangeImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
};
class WriteConflictRangeImpl : public SpecialKeyRangeBaseImpl {
class WriteConflictRangeImpl : public SpecialKeyRangeReadImpl {
public:
explicit WriteConflictRangeImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
@ -196,5 +231,47 @@ public:
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
};
class ManagementCommandsOptionsImpl : public SpecialKeyRangeRWImpl {
public:
explicit ManagementCommandsOptionsImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
Key decode(const KeyRef& key) const override;
Key encode(const KeyRef& key) const override;
Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
};
class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl {
public:
explicit ExcludeServersRangeImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
Key decode(const KeyRef& key) const override;
Key encode(const KeyRef& key) const override;
Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
};
class FailedServersRangeImpl : public SpecialKeyRangeRWImpl {
public:
explicit FailedServersRangeImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
Key decode(const KeyRef& key) const override;
Key encode(const KeyRef& key) const override;
Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
};
class ExclusionInProgressRangeImpl : public SpecialKeyRangeAsyncImpl {
public:
explicit ExclusionInProgressRangeImpl(KeyRangeRef kr);
Future<Standalone<RangeResultRef>> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override;
};
#include "flow/unactorcompiler.h"
#endif

23
fdbclient/StorageServerInterface.h
View File

@ -434,9 +434,30 @@ struct SplitMetricsRequest {
}
};
// Should always be used inside a `Standalone`.
struct ReadHotRangeWithMetrics {
KeyRangeRef keys;
double density;
double readBandwidth;
ReadHotRangeWithMetrics() = default;
ReadHotRangeWithMetrics(KeyRangeRef const& keys, double density, double readBandwidth)
: keys(keys), density(density), readBandwidth(readBandwidth) {}
ReadHotRangeWithMetrics(Arena& arena, const ReadHotRangeWithMetrics& rhs)
: keys(arena, rhs.keys), density(rhs.density), readBandwidth(rhs.readBandwidth) {}
int expectedSize() { return keys.expectedSize() + sizeof(density) + sizeof(readBandwidth); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, keys, density, readBandwidth);
}
};
struct ReadHotSubRangeReply {
constexpr static FileIdentifier file_identifier = 10424537;
Standalone<VectorRef<KeyRangeRef>> readHotRanges;
Standalone<VectorRef<ReadHotRangeWithMetrics>> readHotRanges;
template <class Ar>
void serialize(Ar& ar) {

4
fdbclient/TagThrottle.actor.cpp
View File

@ -339,7 +339,7 @@ namespace ThrottleApi {
loop {
try {
Optional<Value> value = wait(tr.get(tagThrottleAutoEnabledKey));
if(!value.present() || (enabled && value.get() != LiteralStringRef("1") || (!enabled && value.get() != LiteralStringRef("0")))) {
if (!value.present() || (enabled && value.get() != LiteralStringRef("1")) || (!enabled && value.get() != LiteralStringRef("0"))) {
tr.set(tagThrottleAutoEnabledKey, LiteralStringRef(enabled ? "1" : "0"));
signalThrottleChange(tr);
@ -352,4 +352,4 @@ namespace ThrottleApi {
}
}
}
}
}

5
fdbclient/TagThrottle.h
View File

@ -54,7 +54,10 @@ public:
const_iterator end() const {
return tags.end();
}
void clear() {
tags.clear();
bytes = 0;
}
//private:
Arena arena;
std::set<TransactionTagRef> tags;

2
fdbclient/ThreadSafeTransaction.actor.cpp → fdbclient/ThreadSafeTransaction.cpp
View File

@ -1,5 +1,5 @@
/*
* ThreadSafeTransaction.actor.cpp
* ThreadSafeTransaction.cpp
*
* This source file is part of the FoundationDB open source project
*

2
fdbclient/vexillographer/fdb.options
View File

@ -264,6 +264,8 @@ description is not currently required but encouraged.
description="The transaction can retrieve keys that are conflicting with other transactions." />
<Option name="special_key_space_relaxed" code="713"
description="By default, the special key space will only allow users to read from exactly one module (a subspace in the special key space). Use this option to allow reading from zero or more modules. Users who set this option should be prepared for new modules, which may have different behaviors than the modules they're currently reading. For example, a new module might block or return an error." />
<Option name="special_key_space_enable_writes" code="714"
description="By default, users are not allowed to write to special keys. Enable this option will implicitly enable all options required to achieve the configuration change." />
<Option name="tag" code="800" paramType="String" paramDescription="String identifier used to associated this transaction with a throttling group. Must not exceed 16 characters."
description="Adds a tag to the transaction that can be used to apply manual targeted throttling. At most 5 tags can be set on a transaction." />
<Option name="auto_throttle_tag" code="801" paramType="String" paramDescription="String identifier used to associated this transaction with a throttling group. Must not exceed 16 characters."

15
fdbrpc/CMakeLists.txt
View File

@ -31,20 +31,7 @@ set(FDBRPC_SRCS
set(FDBRPC_THIRD_PARTY_SRCS
libcoroutine/Common.c
libcoroutine/Coro.c
zlib/adler32.c
zlib/crc32.c
zlib/deflate.c
zlib/gzclose.c
zlib/gzlib.c
zlib/gzread.c
zlib/gzwrite.c
zlib/infback.c
zlib/inffast.c
zlib/inflate.c
zlib/inftrees.c
zlib/trees.c
zlib/zutil.c)
libcoroutine/Coro.c)
if(APPLE)
list(APPEND FDBRPC_THIRD_PARTY_SRCS libcoroutine/asm.S)

10
fdbrpc/FlowTransport.actor.cpp
View File

@ -409,12 +409,16 @@ ACTOR Future<Void> connectionWriter( Reference<Peer> self, Reference<IConnection
loop {
lastWriteTime = now();
int sent = conn->write(self->unsent.getUnsent(), /* limit= */ FLOW_KNOBS->MAX_PACKET_SEND_BYTES);
if (sent) {
int sent = conn->write(self->unsent.getUnsent(), FLOW_KNOBS->MAX_PACKET_SEND_BYTES);
if (sent != 0) {
self->transport->bytesSent += sent;
self->unsent.sent(sent);
}
if (self->unsent.empty()) break;
if (self->unsent.empty()) {
break;
}
TEST(true); // We didn't write everything, so apparently the write buffer is full. Wait for it to be nonfull.
wait( conn->onWritable() );

27
fdbrpc/sim2.actor.cpp
View File

@ -1217,21 +1217,40 @@ public:
if (tooManyDead) {
newKt = Reboot;
canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("TLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy validates against dead processes.");
TraceEvent("KillChanged")
.detail("KillType", kt)
.detail("NewKillType", newKt)
.detail("TLogPolicy", tLogPolicy->info())
.detail("Reason", "Too many dead processes that cannot satisfy tLogPolicy.");
}
// Reboot and Delete if remaining machines do NOT fulfill policies
else if ((kt < RebootAndDelete) && notEnoughLeft) {
newKt = RebootAndDelete;
canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("TLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy does not validates against remaining processes.");
TraceEvent("KillChanged")
.detail("KillType", kt)
.detail("NewKillType", newKt)
.detail("TLogPolicy", tLogPolicy->info())
.detail("Reason", "Not enough tLog left to satisfy tLogPolicy.");
}
else if ((kt < RebootAndDelete) && (nQuorum > uniqueMachines.size())) {
newKt = RebootAndDelete;
canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("StoragePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
TraceEvent("KillChanged")
.detail("KillType", kt)
.detail("NewKillType", newKt)
.detail("StoragePolicy", storagePolicy->info())
.detail("Quorum", nQuorum)
.detail("Machines", uniqueMachines.size())
.detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
}
else {
TraceEvent("CanSurviveKills").detail("KillType", kt).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size());
TraceEvent("CanSurviveKills")
.detail("KillType", kt)
.detail("TLogPolicy", tLogPolicy->info())
.detail("StoragePolicy", storagePolicy->info())
.detail("Quorum", nQuorum)
.detail("Machines", uniqueMachines.size());
}
}
if (newKillType) *newKillType = newKt;

179
fdbrpc/zlib/adler32.c
View File

@ -1,179 +0,0 @@
/* adler32.c -- compute the Adler-32 checksum of a data stream
* Copyright (C) 1995-2011 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* @(#) $Id$ */
#include "zutil.h"
#define local static
local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
#define BASE 65521 /* largest prime smaller than 65536 */
#define NMAX 5552
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
#define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;}
#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1);
#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2);
#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4);
#define DO16(buf) DO8(buf,0); DO8(buf,8);
/* use NO_DIVIDE if your processor does not do division in hardware --
try it both ways to see which is faster */
#ifdef NO_DIVIDE
/* note that this assumes BASE is 65521, where 65536 % 65521 == 15
(thank you to John Reiser for pointing this out) */
# define CHOP(a) \
do { \
unsigned long tmp = a >> 16; \
a &= 0xffffUL; \
a += (tmp << 4) - tmp; \
} while (0)
# define MOD28(a) \
do { \
CHOP(a); \
if (a >= BASE) a -= BASE; \
} while (0)
# define MOD(a) \
do { \
CHOP(a); \
MOD28(a); \
} while (0)
# define MOD63(a) \
do { /* this assumes a is not negative */ \
z_off64_t tmp = a >> 32; \
a &= 0xffffffffL; \
a += (tmp << 8) - (tmp << 5) + tmp; \
tmp = a >> 16; \
a &= 0xffffL; \
a += (tmp << 4) - tmp; \
tmp = a >> 16; \
a &= 0xffffL; \
a += (tmp << 4) - tmp; \
if (a >= BASE) a -= BASE; \
} while (0)
#else
# define MOD(a) a %= BASE
# define MOD28(a) a %= BASE
# define MOD63(a) a %= BASE
#endif
/* ========================================================================= */
uLong ZEXPORT adler32(adler, buf, len)
uLong adler;
const Bytef *buf;
uInt len;
{
unsigned long sum2;
unsigned n;
/* split Adler-32 into component sums */
sum2 = (adler >> 16) & 0xffff;
adler &= 0xffff;
/* in case user likes doing a byte at a time, keep it fast */
if (len == 1) {
adler += buf[0];
if (adler >= BASE)
adler -= BASE;
sum2 += adler;
if (sum2 >= BASE)
sum2 -= BASE;
return adler | (sum2 << 16);
}
/* initial Adler-32 value (deferred check for len == 1 speed) */
if (buf == Z_NULL)
return 1L;
/* in case short lengths are provided, keep it somewhat fast */
if (len < 16) {
while (len--) {
adler += *buf++;
sum2 += adler;
}
if (adler >= BASE)
adler -= BASE;
MOD28(sum2); /* only added so many BASE's */
return adler | (sum2 << 16);
}
/* do length NMAX blocks -- requires just one modulo operation */
while (len >= NMAX) {
len -= NMAX;
n = NMAX / 16; /* NMAX is divisible by 16 */
do {
DO16(buf); /* 16 sums unrolled */
buf += 16;
} while (--n);
MOD(adler);
MOD(sum2);
}
/* do remaining bytes (less than NMAX, still just one modulo) */
if (len) { /* avoid modulos if none remaining */
while (len >= 16) {
len -= 16;
DO16(buf);
buf += 16;
}
while (len--) {
adler += *buf++;
sum2 += adler;
}
MOD(adler);
MOD(sum2);
}
/* return recombined sums */
return adler | (sum2 << 16);
}
/* ========================================================================= */
local uLong adler32_combine_(adler1, adler2, len2)
uLong adler1;
uLong adler2;
z_off64_t len2;
{
unsigned long sum1;
unsigned long sum2;
unsigned rem;
/* for negative len, return invalid adler32 as a clue for debugging */
if (len2 < 0)
return 0xffffffffUL;
/* the derivation of this formula is left as an exercise for the reader */
MOD63(len2); /* assumes len2 >= 0 */
rem = (unsigned)len2;
sum1 = adler1 & 0xffff;
sum2 = rem * sum1;
MOD(sum2);
sum1 += (adler2 & 0xffff) + BASE - 1;
sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem;
if (sum1 >= BASE) sum1 -= BASE;
if (sum1 >= BASE) sum1 -= BASE;
if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1);
if (sum2 >= BASE) sum2 -= BASE;
return sum1 | (sum2 << 16);
}
/* ========================================================================= */
uLong ZEXPORT adler32_combine(adler1, adler2, len2)
uLong adler1;
uLong adler2;
z_off_t len2;
{
return adler32_combine_(adler1, adler2, len2);
}
uLong ZEXPORT adler32_combine64(adler1, adler2, len2)
uLong adler1;
uLong adler2;
z_off64_t len2;
{
return adler32_combine_(adler1, adler2, len2);
}

425
fdbrpc/zlib/crc32.c
View File

@ -1,425 +0,0 @@
/* crc32.c -- compute the CRC-32 of a data stream
* Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*
* Thanks to Rodney Brown <rbrown64@csc.com.au> for his contribution of faster
* CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing
* tables for updating the shift register in one step with three exclusive-ors
* instead of four steps with four exclusive-ors. This results in about a
* factor of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3.
*/
/* @(#) $Id$ */
/*
Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore
protection on the static variables used to control the first-use generation
of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should
first call get_crc_table() to initialize the tables before allowing more than
one thread to use crc32().
DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h.
*/
#ifdef MAKECRCH
# include <stdio.h>
# ifndef DYNAMIC_CRC_TABLE
# define DYNAMIC_CRC_TABLE
# endif /* !DYNAMIC_CRC_TABLE */
#endif /* MAKECRCH */
#include "zutil.h" /* for STDC and FAR definitions */
#define local static
/* Definitions for doing the crc four data bytes at a time. */
#if !defined(NOBYFOUR) && defined(Z_U4)
# define BYFOUR
#endif
#ifdef BYFOUR
local unsigned long crc32_little OF((unsigned long,
const unsigned char FAR *, unsigned));
local unsigned long crc32_big OF((unsigned long,
const unsigned char FAR *, unsigned));
# define TBLS 8
#else
# define TBLS 1
#endif /* BYFOUR */
/* Local functions for crc concatenation */
local unsigned long gf2_matrix_times OF((unsigned long *mat,
unsigned long vec));
local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat));
local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2));
#ifdef DYNAMIC_CRC_TABLE
local volatile int crc_table_empty = 1;
local z_crc_t FAR crc_table[TBLS][256];
local void make_crc_table OF((void));
#ifdef MAKECRCH
local void write_table OF((FILE *, const z_crc_t FAR *));
#endif /* MAKECRCH */
/*
Generate tables for a byte-wise 32-bit CRC calculation on the polynomial:
x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1.
Polynomials over GF(2) are represented in binary, one bit per coefficient,
with the lowest powers in the most significant bit. Then adding polynomials
is just exclusive-or, and multiplying a polynomial by x is a right shift by
one. If we call the above polynomial p, and represent a byte as the
polynomial q, also with the lowest power in the most significant bit (so the
byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p,
where a mod b means the remainder after dividing a by b.
This calculation is done using the shift-register method of multiplying and
taking the remainder. The register is initialized to zero, and for each
incoming bit, x^32 is added mod p to the register if the bit is a one (where
x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by
x (which is shifting right by one and adding x^32 mod p if the bit shifted
out is a one). We start with the highest power (least significant bit) of
q and repeat for all eight bits of q.
The first table is simply the CRC of all possible eight bit values. This is
all the information needed to generate CRCs on data a byte at a time for all
combinations of CRC register values and incoming bytes. The remaining tables
allow for word-at-a-time CRC calculation for both big-endian and little-
endian machines, where a word is four bytes.
*/
local void make_crc_table()
{
z_crc_t c;
int n, k;
z_crc_t poly; /* polynomial exclusive-or pattern */
/* terms of polynomial defining this crc (except x^32): */
static volatile int first = 1; /* flag to limit concurrent making */
static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26};
/* See if another task is already doing this (not thread-safe, but better
than nothing -- significantly reduces duration of vulnerability in
case the advice about DYNAMIC_CRC_TABLE is ignored) */
if (first) {
first = 0;
/* make exclusive-or pattern from polynomial (0xedb88320UL) */
poly = 0;
for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++)
poly |= (z_crc_t)1 << (31 - p[n]);
/* generate a crc for every 8-bit value */
for (n = 0; n < 256; n++) {
c = (z_crc_t)n;
for (k = 0; k < 8; k++)
c = c & 1 ? poly ^ (c >> 1) : c >> 1;
crc_table[0][n] = c;
}
#ifdef BYFOUR
/* generate crc for each value followed by one, two, and three zeros,
and then the byte reversal of those as well as the first table */
for (n = 0; n < 256; n++) {
c = crc_table[0][n];
crc_table[4][n] = ZSWAP32(c);
for (k = 1; k < 4; k++) {
c = crc_table[0][c & 0xff] ^ (c >> 8);
crc_table[k][n] = c;
crc_table[k + 4][n] = ZSWAP32(c);
}
}
#endif /* BYFOUR */
crc_table_empty = 0;
}
else { /* not first */
/* wait for the other guy to finish (not efficient, but rare) */
while (crc_table_empty)
;
}
#ifdef MAKECRCH
/* write out CRC tables to crc32.h */
{
FILE *out;
out = fopen("crc32.h", "w");
if (out == NULL) return;
fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n");
fprintf(out, " * Generated automatically by crc32.c\n */\n\n");
fprintf(out, "local const z_crc_t FAR ");
fprintf(out, "crc_table[TBLS][256] =\n{\n {\n");
write_table(out, crc_table[0]);
# ifdef BYFOUR
fprintf(out, "#ifdef BYFOUR\n");
for (k = 1; k < 8; k++) {
fprintf(out, " },\n {\n");
write_table(out, crc_table[k]);
}
fprintf(out, "#endif\n");
# endif /* BYFOUR */
fprintf(out, " }\n};\n");
fclose(out);
}
#endif /* MAKECRCH */
}
#ifdef MAKECRCH
local void write_table(out, table)
FILE *out;
const z_crc_t FAR *table;
{
int n;
for (n = 0; n < 256; n++)
fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ",
(unsigned long)(table[n]),
n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", "));
}
#endif /* MAKECRCH */
#else /* !DYNAMIC_CRC_TABLE */
/* ========================================================================
* Tables of CRC-32s of all single-byte values, made by make_crc_table().
*/
#include "crc32.h"
#endif /* DYNAMIC_CRC_TABLE */
/* =========================================================================
* This function can be used by asm versions of crc32()
*/
const z_crc_t FAR * ZEXPORT get_crc_table()
{
#ifdef DYNAMIC_CRC_TABLE
if (crc_table_empty)
make_crc_table();
#endif /* DYNAMIC_CRC_TABLE */
return (const z_crc_t FAR *)crc_table;
}
/* ========================================================================= */
#define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8)
#define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1
/* ========================================================================= */
unsigned long ZEXPORT crc32(crc, buf, len)
unsigned long crc;
const unsigned char FAR *buf;
uInt len;
{
if (buf == Z_NULL) return 0UL;
#ifdef DYNAMIC_CRC_TABLE
if (crc_table_empty)
make_crc_table();
#endif /* DYNAMIC_CRC_TABLE */
#ifdef BYFOUR
if (sizeof(void *) == sizeof(ptrdiff_t)) {
z_crc_t endian;
endian = 1;
if (*((unsigned char *)(&endian)))
return crc32_little(crc, buf, len);
else
return crc32_big(crc, buf, len);
}
#endif /* BYFOUR */
crc = crc ^ 0xffffffffUL;
while (len >= 8) {
DO8;
len -= 8;
}
if (len) do {
DO1;
} while (--len);
return crc ^ 0xffffffffUL;
}
#ifdef BYFOUR
/* ========================================================================= */
#define DOLIT4 c ^= *buf4++; \
c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \
crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24]
#define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4
/* ========================================================================= */
local unsigned long crc32_little(crc, buf, len)
unsigned long crc;
const unsigned char FAR *buf;
unsigned len;
{
register z_crc_t c;
register const z_crc_t FAR *buf4;
c = (z_crc_t)crc;
c = ~c;
while (len && ((ptrdiff_t)buf & 3)) {
c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8);
len--;
}
buf4 = (const z_crc_t FAR *)(const void FAR *)buf;
while (len >= 32) {
DOLIT32;
len -= 32;
}
while (len >= 4) {
DOLIT4;
len -= 4;
}
buf = (const unsigned char FAR *)buf4;
if (len) do {
c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8);
} while (--len);
c = ~c;
return (unsigned long)c;
}
/* ========================================================================= */
#define DOBIG4 c ^= *++buf4; \
c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \
crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24]
#define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4
/* ========================================================================= */
local unsigned long crc32_big(crc, buf, len)
unsigned long crc;
const unsigned char FAR *buf;
unsigned len;
{
register z_crc_t c;
register const z_crc_t FAR *buf4;
c = ZSWAP32((z_crc_t)crc);
c = ~c;
while (len && ((ptrdiff_t)buf & 3)) {
c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8);
len--;
}
buf4 = (const z_crc_t FAR *)(const void FAR *)buf;
buf4--;
while (len >= 32) {
DOBIG32;
len -= 32;
}
while (len >= 4) {
DOBIG4;
len -= 4;
}
buf4++;
buf = (const unsigned char FAR *)buf4;
if (len) do {
c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8);
} while (--len);
c = ~c;
return (unsigned long)(ZSWAP32(c));
}
#endif /* BYFOUR */
#define GF2_DIM 32 /* dimension of GF(2) vectors (length of CRC) */
/* ========================================================================= */
local unsigned long gf2_matrix_times(mat, vec)
unsigned long *mat;
unsigned long vec;
{
unsigned long sum;
sum = 0;
while (vec) {
if (vec & 1)
sum ^= *mat;
vec >>= 1;
mat++;
}
return sum;
}
/* ========================================================================= */
local void gf2_matrix_square(square, mat)
unsigned long *square;
unsigned long *mat;
{
int n;
for (n = 0; n < GF2_DIM; n++)
square[n] = gf2_matrix_times(mat, mat[n]);
}
/* ========================================================================= */
local uLong crc32_combine_(crc1, crc2, len2)
uLong crc1;
uLong crc2;
z_off64_t len2;
{
int n;
unsigned long row;
unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */
unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */
/* degenerate case (also disallow negative lengths) */
if (len2 <= 0)
return crc1;
/* put operator for one zero bit in odd */
odd[0] = 0xedb88320UL; /* CRC-32 polynomial */
row = 1;
for (n = 1; n < GF2_DIM; n++) {
odd[n] = row;
row <<= 1;
}
/* put operator for two zero bits in even */
gf2_matrix_square(even, odd);
/* put operator for four zero bits in odd */
gf2_matrix_square(odd, even);
/* apply len2 zeros to crc1 (first square will put the operator for one
zero byte, eight zero bits, in even) */
do {
/* apply zeros operator for this bit of len2 */
gf2_matrix_square(even, odd);
if (len2 & 1)
crc1 = gf2_matrix_times(even, crc1);
len2 >>= 1;
/* if no more bits set, then done */
if (len2 == 0)
break;
/* another iteration of the loop with odd and even swapped */
gf2_matrix_square(odd, even);
if (len2 & 1)
crc1 = gf2_matrix_times(odd, crc1);
len2 >>= 1;
/* if no more bits set, then done */
} while (len2 != 0);
/* return combined crc */
crc1 ^= crc2;
return crc1;
}
/* ========================================================================= */
uLong ZEXPORT crc32_combine(crc1, crc2, len2)
uLong crc1;
uLong crc2;
z_off_t len2;
{
return crc32_combine_(crc1, crc2, len2);
}
uLong ZEXPORT crc32_combine64(crc1, crc2, len2)
uLong crc1;
uLong crc2;
z_off64_t len2;
{
return crc32_combine_(crc1, crc2, len2);
}

441
fdbrpc/zlib/crc32.h
View File

@ -1,441 +0,0 @@
/* crc32.h -- tables for rapid CRC calculation
* Generated automatically by crc32.c
*/
local const z_crc_t FAR crc_table[TBLS][256] =
{
{
0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL,
0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL,
0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL,
0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL,
0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL,
0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL,
0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL,
0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL,
0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL,
0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL,
0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL,
0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL,
0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL,
0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL,
0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL,
0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL,
0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL,
0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL,
0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL,
0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL,
0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL,
0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL,
0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL,
0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL,
0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL,
0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL,
0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL,
0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL,
0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL,
0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL,
0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL,
0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL,
0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL,
0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL,
0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL,
0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL,
0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL,
0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL,
0x2d02ef8dUL
#ifdef BYFOUR
},
{
0x00000000UL, 0x191b3141UL, 0x32366282UL, 0x2b2d53c3UL, 0x646cc504UL,
0x7d77f445UL, 0x565aa786UL, 0x4f4196c7UL, 0xc8d98a08UL, 0xd1c2bb49UL,
0xfaefe88aUL, 0xe3f4d9cbUL, 0xacb54f0cUL, 0xb5ae7e4dUL, 0x9e832d8eUL,
0x87981ccfUL, 0x4ac21251UL, 0x53d92310UL, 0x78f470d3UL, 0x61ef4192UL,
0x2eaed755UL, 0x37b5e614UL, 0x1c98b5d7UL, 0x05838496UL, 0x821b9859UL,
0x9b00a918UL, 0xb02dfadbUL, 0xa936cb9aUL, 0xe6775d5dUL, 0xff6c6c1cUL,
0xd4413fdfUL, 0xcd5a0e9eUL, 0x958424a2UL, 0x8c9f15e3UL, 0xa7b24620UL,
0xbea97761UL, 0xf1e8e1a6UL, 0xe8f3d0e7UL, 0xc3de8324UL, 0xdac5b265UL,
0x5d5daeaaUL, 0x44469febUL, 0x6f6bcc28UL, 0x7670fd69UL, 0x39316baeUL,
0x202a5aefUL, 0x0b07092cUL, 0x121c386dUL, 0xdf4636f3UL, 0xc65d07b2UL,
0xed705471UL, 0xf46b6530UL, 0xbb2af3f7UL, 0xa231c2b6UL, 0x891c9175UL,
0x9007a034UL, 0x179fbcfbUL, 0x0e848dbaUL, 0x25a9de79UL, 0x3cb2ef38UL,
0x73f379ffUL, 0x6ae848beUL, 0x41c51b7dUL, 0x58de2a3cUL, 0xf0794f05UL,
0xe9627e44UL, 0xc24f2d87UL, 0xdb541cc6UL, 0x94158a01UL, 0x8d0ebb40UL,
0xa623e883UL, 0xbf38d9c2UL, 0x38a0c50dUL, 0x21bbf44cUL, 0x0a96a78fUL,
0x138d96ceUL, 0x5ccc0009UL, 0x45d73148UL, 0x6efa628bUL, 0x77e153caUL,
0xbabb5d54UL, 0xa3a06c15UL, 0x888d3fd6UL, 0x91960e97UL, 0xded79850UL,
0xc7cca911UL, 0xece1fad2UL, 0xf5facb93UL, 0x7262d75cUL, 0x6b79e61dUL,
0x4054b5deUL, 0x594f849fUL, 0x160e1258UL, 0x0f152319UL, 0x243870daUL,
0x3d23419bUL, 0x65fd6ba7UL, 0x7ce65ae6UL, 0x57cb0925UL, 0x4ed03864UL,
0x0191aea3UL, 0x188a9fe2UL, 0x33a7cc21UL, 0x2abcfd60UL, 0xad24e1afUL,
0xb43fd0eeUL, 0x9f12832dUL, 0x8609b26cUL, 0xc94824abUL, 0xd05315eaUL,
0xfb7e4629UL, 0xe2657768UL, 0x2f3f79f6UL, 0x362448b7UL, 0x1d091b74UL,
0x04122a35UL, 0x4b53bcf2UL, 0x52488db3UL, 0x7965de70UL, 0x607eef31UL,
0xe7e6f3feUL, 0xfefdc2bfUL, 0xd5d0917cUL, 0xcccba03dUL, 0x838a36faUL,
0x9a9107bbUL, 0xb1bc5478UL, 0xa8a76539UL, 0x3b83984bUL, 0x2298a90aUL,
0x09b5fac9UL, 0x10aecb88UL, 0x5fef5d4fUL, 0x46f46c0eUL, 0x6dd93fcdUL,
0x74c20e8cUL, 0xf35a1243UL, 0xea412302UL, 0xc16c70c1UL, 0xd8774180UL,
0x9736d747UL, 0x8e2de606UL, 0xa500b5c5UL, 0xbc1b8484UL, 0x71418a1aUL,
0x685abb5bUL, 0x4377e898UL, 0x5a6cd9d9UL, 0x152d4f1eUL, 0x0c367e5fUL,
0x271b2d9cUL, 0x3e001cddUL, 0xb9980012UL, 0xa0833153UL, 0x8bae6290UL,
0x92b553d1UL, 0xddf4c516UL, 0xc4eff457UL, 0xefc2a794UL, 0xf6d996d5UL,
0xae07bce9UL, 0xb71c8da8UL, 0x9c31de6bUL, 0x852aef2aUL, 0xca6b79edUL,
0xd37048acUL, 0xf85d1b6fUL, 0xe1462a2eUL, 0x66de36e1UL, 0x7fc507a0UL,
0x54e85463UL, 0x4df36522UL, 0x02b2f3e5UL, 0x1ba9c2a4UL, 0x30849167UL,
0x299fa026UL, 0xe4c5aeb8UL, 0xfdde9ff9UL, 0xd6f3cc3aUL, 0xcfe8fd7bUL,
0x80a96bbcUL, 0x99b25afdUL, 0xb29f093eUL, 0xab84387fUL, 0x2c1c24b0UL,
0x350715f1UL, 0x1e2a4632UL, 0x07317773UL, 0x4870e1b4UL, 0x516bd0f5UL,
0x7a468336UL, 0x635db277UL, 0xcbfad74eUL, 0xd2e1e60fUL, 0xf9ccb5ccUL,
0xe0d7848dUL, 0xaf96124aUL, 0xb68d230bUL, 0x9da070c8UL, 0x84bb4189UL,
0x03235d46UL, 0x1a386c07UL, 0x31153fc4UL, 0x280e0e85UL, 0x674f9842UL,
0x7e54a903UL, 0x5579fac0UL, 0x4c62cb81UL, 0x8138c51fUL, 0x9823f45eUL,
0xb30ea79dUL, 0xaa1596dcUL, 0xe554001bUL, 0xfc4f315aUL, 0xd7626299UL,
0xce7953d8UL, 0x49e14f17UL, 0x50fa7e56UL, 0x7bd72d95UL, 0x62cc1cd4UL,
0x2d8d8a13UL, 0x3496bb52UL, 0x1fbbe891UL, 0x06a0d9d0UL, 0x5e7ef3ecUL,
0x4765c2adUL, 0x6c48916eUL, 0x7553a02fUL, 0x3a1236e8UL, 0x230907a9UL,
0x0824546aUL, 0x113f652bUL, 0x96a779e4UL, 0x8fbc48a5UL, 0xa4911b66UL,
0xbd8a2a27UL, 0xf2cbbce0UL, 0xebd08da1UL, 0xc0fdde62UL, 0xd9e6ef23UL,
0x14bce1bdUL, 0x0da7d0fcUL, 0x268a833fUL, 0x3f91b27eUL, 0x70d024b9UL,
0x69cb15f8UL, 0x42e6463bUL, 0x5bfd777aUL, 0xdc656bb5UL, 0xc57e5af4UL,
0xee530937UL, 0xf7483876UL, 0xb809aeb1UL, 0xa1129ff0UL, 0x8a3fcc33UL,
0x9324fd72UL
},
{
0x00000000UL, 0x01c26a37UL, 0x0384d46eUL, 0x0246be59UL, 0x0709a8dcUL,
0x06cbc2ebUL, 0x048d7cb2UL, 0x054f1685UL, 0x0e1351b8UL, 0x0fd13b8fUL,
0x0d9785d6UL, 0x0c55efe1UL, 0x091af964UL, 0x08d89353UL, 0x0a9e2d0aUL,
0x0b5c473dUL, 0x1c26a370UL, 0x1de4c947UL, 0x1fa2771eUL, 0x1e601d29UL,
0x1b2f0bacUL, 0x1aed619bUL, 0x18abdfc2UL, 0x1969b5f5UL, 0x1235f2c8UL,
0x13f798ffUL, 0x11b126a6UL, 0x10734c91UL, 0x153c5a14UL, 0x14fe3023UL,
0x16b88e7aUL, 0x177ae44dUL, 0x384d46e0UL, 0x398f2cd7UL, 0x3bc9928eUL,
0x3a0bf8b9UL, 0x3f44ee3cUL, 0x3e86840bUL, 0x3cc03a52UL, 0x3d025065UL,
0x365e1758UL, 0x379c7d6fUL, 0x35dac336UL, 0x3418a901UL, 0x3157bf84UL,
0x3095d5b3UL, 0x32d36beaUL, 0x331101ddUL, 0x246be590UL, 0x25a98fa7UL,
0x27ef31feUL, 0x262d5bc9UL, 0x23624d4cUL, 0x22a0277bUL, 0x20e69922UL,
0x2124f315UL, 0x2a78b428UL, 0x2bbade1fUL, 0x29fc6046UL, 0x283e0a71UL,
0x2d711cf4UL, 0x2cb376c3UL, 0x2ef5c89aUL, 0x2f37a2adUL, 0x709a8dc0UL,
0x7158e7f7UL, 0x731e59aeUL, 0x72dc3399UL, 0x7793251cUL, 0x76514f2bUL,
0x7417f172UL, 0x75d59b45UL, 0x7e89dc78UL, 0x7f4bb64fUL, 0x7d0d0816UL,
0x7ccf6221UL, 0x798074a4UL, 0x78421e93UL, 0x7a04a0caUL, 0x7bc6cafdUL,
0x6cbc2eb0UL, 0x6d7e4487UL, 0x6f38fadeUL, 0x6efa90e9UL, 0x6bb5866cUL,
0x6a77ec5bUL, 0x68315202UL, 0x69f33835UL, 0x62af7f08UL, 0x636d153fUL,
0x612bab66UL, 0x60e9c151UL, 0x65a6d7d4UL, 0x6464bde3UL, 0x662203baUL,
0x67e0698dUL, 0x48d7cb20UL, 0x4915a117UL, 0x4b531f4eUL, 0x4a917579UL,
0x4fde63fcUL, 0x4e1c09cbUL, 0x4c5ab792UL, 0x4d98dda5UL, 0x46c49a98UL,
0x4706f0afUL, 0x45404ef6UL, 0x448224c1UL, 0x41cd3244UL, 0x400f5873UL,
0x4249e62aUL, 0x438b8c1dUL, 0x54f16850UL, 0x55330267UL, 0x5775bc3eUL,
0x56b7d609UL, 0x53f8c08cUL, 0x523aaabbUL, 0x507c14e2UL, 0x51be7ed5UL,
0x5ae239e8UL, 0x5b2053dfUL, 0x5966ed86UL, 0x58a487b1UL, 0x5deb9134UL,
0x5c29fb03UL, 0x5e6f455aUL, 0x5fad2f6dUL, 0xe1351b80UL, 0xe0f771b7UL,
0xe2b1cfeeUL, 0xe373a5d9UL, 0xe63cb35cUL, 0xe7fed96bUL, 0xe5b86732UL,
0xe47a0d05UL, 0xef264a38UL, 0xeee4200fUL, 0xeca29e56UL, 0xed60f461UL,
0xe82fe2e4UL, 0xe9ed88d3UL, 0xebab368aUL, 0xea695cbdUL, 0xfd13b8f0UL,
0xfcd1d2c7UL, 0xfe976c9eUL, 0xff5506a9UL, 0xfa1a102cUL, 0xfbd87a1bUL,
0xf99ec442UL, 0xf85cae75UL, 0xf300e948UL, 0xf2c2837fUL, 0xf0843d26UL,
0xf1465711UL, 0xf4094194UL, 0xf5cb2ba3UL, 0xf78d95faUL, 0xf64fffcdUL,
0xd9785d60UL, 0xd8ba3757UL, 0xdafc890eUL, 0xdb3ee339UL, 0xde71f5bcUL,
0xdfb39f8bUL, 0xddf521d2UL, 0xdc374be5UL, 0xd76b0cd8UL, 0xd6a966efUL,
0xd4efd8b6UL, 0xd52db281UL, 0xd062a404UL, 0xd1a0ce33UL, 0xd3e6706aUL,
0xd2241a5dUL, 0xc55efe10UL, 0xc49c9427UL, 0xc6da2a7eUL, 0xc7184049UL,
0xc25756ccUL, 0xc3953cfbUL, 0xc1d382a2UL, 0xc011e895UL, 0xcb4dafa8UL,
0xca8fc59fUL, 0xc8c97bc6UL, 0xc90b11f1UL, 0xcc440774UL, 0xcd866d43UL,
0xcfc0d31aUL, 0xce02b92dUL, 0x91af9640UL, 0x906dfc77UL, 0x922b422eUL,
0x93e92819UL, 0x96a63e9cUL, 0x976454abUL, 0x9522eaf2UL, 0x94e080c5UL,
0x9fbcc7f8UL, 0x9e7eadcfUL, 0x9c381396UL, 0x9dfa79a1UL, 0x98b56f24UL,
0x99770513UL, 0x9b31bb4aUL, 0x9af3d17dUL, 0x8d893530UL, 0x8c4b5f07UL,
0x8e0de15eUL, 0x8fcf8b69UL, 0x8a809decUL, 0x8b42f7dbUL, 0x89044982UL,
0x88c623b5UL, 0x839a6488UL, 0x82580ebfUL, 0x801eb0e6UL, 0x81dcdad1UL,
0x8493cc54UL, 0x8551a663UL, 0x8717183aUL, 0x86d5720dUL, 0xa9e2d0a0UL,
0xa820ba97UL, 0xaa6604ceUL, 0xaba46ef9UL, 0xaeeb787cUL, 0xaf29124bUL,
0xad6fac12UL, 0xacadc625UL, 0xa7f18118UL, 0xa633eb2fUL, 0xa4755576UL,
0xa5b73f41UL, 0xa0f829c4UL, 0xa13a43f3UL, 0xa37cfdaaUL, 0xa2be979dUL,
0xb5c473d0UL, 0xb40619e7UL, 0xb640a7beUL, 0xb782cd89UL, 0xb2cddb0cUL,
0xb30fb13bUL, 0xb1490f62UL, 0xb08b6555UL, 0xbbd72268UL, 0xba15485fUL,
0xb853f606UL, 0xb9919c31UL, 0xbcde8ab4UL, 0xbd1ce083UL, 0xbf5a5edaUL,
0xbe9834edUL
},
{
0x00000000UL, 0xb8bc6765UL, 0xaa09c88bUL, 0x12b5afeeUL, 0x8f629757UL,
0x37def032UL, 0x256b5fdcUL, 0x9dd738b9UL, 0xc5b428efUL, 0x7d084f8aUL,
0x6fbde064UL, 0xd7018701UL, 0x4ad6bfb8UL, 0xf26ad8ddUL, 0xe0df7733UL,
0x58631056UL, 0x5019579fUL, 0xe8a530faUL, 0xfa109f14UL, 0x42acf871UL,
0xdf7bc0c8UL, 0x67c7a7adUL, 0x75720843UL, 0xcdce6f26UL, 0x95ad7f70UL,
0x2d111815UL, 0x3fa4b7fbUL, 0x8718d09eUL, 0x1acfe827UL, 0xa2738f42UL,
0xb0c620acUL, 0x087a47c9UL, 0xa032af3eUL, 0x188ec85bUL, 0x0a3b67b5UL,
0xb28700d0UL, 0x2f503869UL, 0x97ec5f0cUL, 0x8559f0e2UL, 0x3de59787UL,
0x658687d1UL, 0xdd3ae0b4UL, 0xcf8f4f5aUL, 0x7733283fUL, 0xeae41086UL,
0x525877e3UL, 0x40edd80dUL, 0xf851bf68UL, 0xf02bf8a1UL, 0x48979fc4UL,
0x5a22302aUL, 0xe29e574fUL, 0x7f496ff6UL, 0xc7f50893UL, 0xd540a77dUL,
0x6dfcc018UL, 0x359fd04eUL, 0x8d23b72bUL, 0x9f9618c5UL, 0x272a7fa0UL,
0xbafd4719UL, 0x0241207cUL, 0x10f48f92UL, 0xa848e8f7UL, 0x9b14583dUL,
0x23a83f58UL, 0x311d90b6UL, 0x89a1f7d3UL, 0x1476cf6aUL, 0xaccaa80fUL,
0xbe7f07e1UL, 0x06c36084UL, 0x5ea070d2UL, 0xe61c17b7UL, 0xf4a9b859UL,
0x4c15df3cUL, 0xd1c2e785UL, 0x697e80e0UL, 0x7bcb2f0eUL, 0xc377486bUL,
0xcb0d0fa2UL, 0x73b168c7UL, 0x6104c729UL, 0xd9b8a04cUL, 0x446f98f5UL,
0xfcd3ff90UL, 0xee66507eUL, 0x56da371bUL, 0x0eb9274dUL, 0xb6054028UL,
0xa4b0efc6UL, 0x1c0c88a3UL, 0x81dbb01aUL, 0x3967d77fUL, 0x2bd27891UL,
0x936e1ff4UL, 0x3b26f703UL, 0x839a9066UL, 0x912f3f88UL, 0x299358edUL,
0xb4446054UL, 0x0cf80731UL, 0x1e4da8dfUL, 0xa6f1cfbaUL, 0xfe92dfecUL,
0x462eb889UL, 0x549b1767UL, 0xec277002UL, 0x71f048bbUL, 0xc94c2fdeUL,
0xdbf98030UL, 0x6345e755UL, 0x6b3fa09cUL, 0xd383c7f9UL, 0xc1366817UL,
0x798a0f72UL, 0xe45d37cbUL, 0x5ce150aeUL, 0x4e54ff40UL, 0xf6e89825UL,
0xae8b8873UL, 0x1637ef16UL, 0x048240f8UL, 0xbc3e279dUL, 0x21e91f24UL,
0x99557841UL, 0x8be0d7afUL, 0x335cb0caUL, 0xed59b63bUL, 0x55e5d15eUL,
0x47507eb0UL, 0xffec19d5UL, 0x623b216cUL, 0xda874609UL, 0xc832e9e7UL,
0x708e8e82UL, 0x28ed9ed4UL, 0x9051f9b1UL, 0x82e4565fUL, 0x3a58313aUL,
0xa78f0983UL, 0x1f336ee6UL, 0x0d86c108UL, 0xb53aa66dUL, 0xbd40e1a4UL,
0x05fc86c1UL, 0x1749292fUL, 0xaff54e4aUL, 0x322276f3UL, 0x8a9e1196UL,
0x982bbe78UL, 0x2097d91dUL, 0x78f4c94bUL, 0xc048ae2eUL, 0xd2fd01c0UL,
0x6a4166a5UL, 0xf7965e1cUL, 0x4f2a3979UL, 0x5d9f9697UL, 0xe523f1f2UL,
0x4d6b1905UL, 0xf5d77e60UL, 0xe762d18eUL, 0x5fdeb6ebUL, 0xc2098e52UL,
0x7ab5e937UL, 0x680046d9UL, 0xd0bc21bcUL, 0x88df31eaUL, 0x3063568fUL,
0x22d6f961UL, 0x9a6a9e04UL, 0x07bda6bdUL, 0xbf01c1d8UL, 0xadb46e36UL,
0x15080953UL, 0x1d724e9aUL, 0xa5ce29ffUL, 0xb77b8611UL, 0x0fc7e174UL,
0x9210d9cdUL, 0x2aacbea8UL, 0x38191146UL, 0x80a57623UL, 0xd8c66675UL,
0x607a0110UL, 0x72cfaefeUL, 0xca73c99bUL, 0x57a4f122UL, 0xef189647UL,
0xfdad39a9UL, 0x45115eccUL, 0x764dee06UL, 0xcef18963UL, 0xdc44268dUL,
0x64f841e8UL, 0xf92f7951UL, 0x41931e34UL, 0x5326b1daUL, 0xeb9ad6bfUL,
0xb3f9c6e9UL, 0x0b45a18cUL, 0x19f00e62UL, 0xa14c6907UL, 0x3c9b51beUL,
0x842736dbUL, 0x96929935UL, 0x2e2efe50UL, 0x2654b999UL, 0x9ee8defcUL,
0x8c5d7112UL, 0x34e11677UL, 0xa9362eceUL, 0x118a49abUL, 0x033fe645UL,
0xbb838120UL, 0xe3e09176UL, 0x5b5cf613UL, 0x49e959fdUL, 0xf1553e98UL,
0x6c820621UL, 0xd43e6144UL, 0xc68bceaaUL, 0x7e37a9cfUL, 0xd67f4138UL,
0x6ec3265dUL, 0x7c7689b3UL, 0xc4caeed6UL, 0x591dd66fUL, 0xe1a1b10aUL,
0xf3141ee4UL, 0x4ba87981UL, 0x13cb69d7UL, 0xab770eb2UL, 0xb9c2a15cUL,
0x017ec639UL, 0x9ca9fe80UL, 0x241599e5UL, 0x36a0360bUL, 0x8e1c516eUL,
0x866616a7UL, 0x3eda71c2UL, 0x2c6fde2cUL, 0x94d3b949UL, 0x090481f0UL,
0xb1b8e695UL, 0xa30d497bUL, 0x1bb12e1eUL, 0x43d23e48UL, 0xfb6e592dUL,
0xe9dbf6c3UL, 0x516791a6UL, 0xccb0a91fUL, 0x740cce7aUL, 0x66b96194UL,
0xde0506f1UL
},
{
0x00000000UL, 0x96300777UL, 0x2c610eeeUL, 0xba510999UL, 0x19c46d07UL,
0x8ff46a70UL, 0x35a563e9UL, 0xa395649eUL, 0x3288db0eUL, 0xa4b8dc79UL,
0x1ee9d5e0UL, 0x88d9d297UL, 0x2b4cb609UL, 0xbd7cb17eUL, 0x072db8e7UL,
0x911dbf90UL, 0x6410b71dUL, 0xf220b06aUL, 0x4871b9f3UL, 0xde41be84UL,
0x7dd4da1aUL, 0xebe4dd6dUL, 0x51b5d4f4UL, 0xc785d383UL, 0x56986c13UL,
0xc0a86b64UL, 0x7af962fdUL, 0xecc9658aUL, 0x4f5c0114UL, 0xd96c0663UL,
0x633d0ffaUL, 0xf50d088dUL, 0xc8206e3bUL, 0x5e10694cUL, 0xe44160d5UL,
0x727167a2UL, 0xd1e4033cUL, 0x47d4044bUL, 0xfd850dd2UL, 0x6bb50aa5UL,
0xfaa8b535UL, 0x6c98b242UL, 0xd6c9bbdbUL, 0x40f9bcacUL, 0xe36cd832UL,
0x755cdf45UL, 0xcf0dd6dcUL, 0x593dd1abUL, 0xac30d926UL, 0x3a00de51UL,
0x8051d7c8UL, 0x1661d0bfUL, 0xb5f4b421UL, 0x23c4b356UL, 0x9995bacfUL,
0x0fa5bdb8UL, 0x9eb80228UL, 0x0888055fUL, 0xb2d90cc6UL, 0x24e90bb1UL,
0x877c6f2fUL, 0x114c6858UL, 0xab1d61c1UL, 0x3d2d66b6UL, 0x9041dc76UL,
0x0671db01UL, 0xbc20d298UL, 0x2a10d5efUL, 0x8985b171UL, 0x1fb5b606UL,
0xa5e4bf9fUL, 0x33d4b8e8UL, 0xa2c90778UL, 0x34f9000fUL, 0x8ea80996UL,
0x18980ee1UL, 0xbb0d6a7fUL, 0x2d3d6d08UL, 0x976c6491UL, 0x015c63e6UL,
0xf4516b6bUL, 0x62616c1cUL, 0xd8306585UL, 0x4e0062f2UL, 0xed95066cUL,
0x7ba5011bUL, 0xc1f40882UL, 0x57c40ff5UL, 0xc6d9b065UL, 0x50e9b712UL,
0xeab8be8bUL, 0x7c88b9fcUL, 0xdf1ddd62UL, 0x492dda15UL, 0xf37cd38cUL,
0x654cd4fbUL, 0x5861b24dUL, 0xce51b53aUL, 0x7400bca3UL, 0xe230bbd4UL,
0x41a5df4aUL, 0xd795d83dUL, 0x6dc4d1a4UL, 0xfbf4d6d3UL, 0x6ae96943UL,
0xfcd96e34UL, 0x468867adUL, 0xd0b860daUL, 0x732d0444UL, 0xe51d0333UL,
0x5f4c0aaaUL, 0xc97c0dddUL, 0x3c710550UL, 0xaa410227UL, 0x10100bbeUL,
0x86200cc9UL, 0x25b56857UL, 0xb3856f20UL, 0x09d466b9UL, 0x9fe461ceUL,
0x0ef9de5eUL, 0x98c9d929UL, 0x2298d0b0UL, 0xb4a8d7c7UL, 0x173db359UL,
0x810db42eUL, 0x3b5cbdb7UL, 0xad6cbac0UL, 0x2083b8edUL, 0xb6b3bf9aUL,
0x0ce2b603UL, 0x9ad2b174UL, 0x3947d5eaUL, 0xaf77d29dUL, 0x1526db04UL,
0x8316dc73UL, 0x120b63e3UL, 0x843b6494UL, 0x3e6a6d0dUL, 0xa85a6a7aUL,
0x0bcf0ee4UL, 0x9dff0993UL, 0x27ae000aUL, 0xb19e077dUL, 0x44930ff0UL,
0xd2a30887UL, 0x68f2011eUL, 0xfec20669UL, 0x5d5762f7UL, 0xcb676580UL,
0x71366c19UL, 0xe7066b6eUL, 0x761bd4feUL, 0xe02bd389UL, 0x5a7ada10UL,
0xcc4add67UL, 0x6fdfb9f9UL, 0xf9efbe8eUL, 0x43beb717UL, 0xd58eb060UL,
0xe8a3d6d6UL, 0x7e93d1a1UL, 0xc4c2d838UL, 0x52f2df4fUL, 0xf167bbd1UL,
0x6757bca6UL, 0xdd06b53fUL, 0x4b36b248UL, 0xda2b0dd8UL, 0x4c1b0aafUL,
0xf64a0336UL, 0x607a0441UL, 0xc3ef60dfUL, 0x55df67a8UL, 0xef8e6e31UL,
0x79be6946UL, 0x8cb361cbUL, 0x1a8366bcUL, 0xa0d26f25UL, 0x36e26852UL,
0x95770cccUL, 0x03470bbbUL, 0xb9160222UL, 0x2f260555UL, 0xbe3bbac5UL,
0x280bbdb2UL, 0x925ab42bUL, 0x046ab35cUL, 0xa7ffd7c2UL, 0x31cfd0b5UL,
0x8b9ed92cUL, 0x1daede5bUL, 0xb0c2649bUL, 0x26f263ecUL, 0x9ca36a75UL,
0x0a936d02UL, 0xa906099cUL, 0x3f360eebUL, 0x85670772UL, 0x13570005UL,
0x824abf95UL, 0x147ab8e2UL, 0xae2bb17bUL, 0x381bb60cUL, 0x9b8ed292UL,
0x0dbed5e5UL, 0xb7efdc7cUL, 0x21dfdb0bUL, 0xd4d2d386UL, 0x42e2d4f1UL,
0xf8b3dd68UL, 0x6e83da1fUL, 0xcd16be81UL, 0x5b26b9f6UL, 0xe177b06fUL,
0x7747b718UL, 0xe65a0888UL, 0x706a0fffUL, 0xca3b0666UL, 0x5c0b0111UL,
0xff9e658fUL, 0x69ae62f8UL, 0xd3ff6b61UL, 0x45cf6c16UL, 0x78e20aa0UL,
0xeed20dd7UL, 0x5483044eUL, 0xc2b30339UL, 0x612667a7UL, 0xf71660d0UL,
0x4d476949UL, 0xdb776e3eUL, 0x4a6ad1aeUL, 0xdc5ad6d9UL, 0x660bdf40UL,
0xf03bd837UL, 0x53aebca9UL, 0xc59ebbdeUL, 0x7fcfb247UL, 0xe9ffb530UL,
0x1cf2bdbdUL, 0x8ac2bacaUL, 0x3093b353UL, 0xa6a3b424UL, 0x0536d0baUL,
0x9306d7cdUL, 0x2957de54UL, 0xbf67d923UL, 0x2e7a66b3UL, 0xb84a61c4UL,
0x021b685dUL, 0x942b6f2aUL, 0x37be0bb4UL, 0xa18e0cc3UL, 0x1bdf055aUL,
0x8def022dUL
},
{
0x00000000UL, 0x41311b19UL, 0x82623632UL, 0xc3532d2bUL, 0x04c56c64UL,
0x45f4777dUL, 0x86a75a56UL, 0xc796414fUL, 0x088ad9c8UL, 0x49bbc2d1UL,
0x8ae8effaUL, 0xcbd9f4e3UL, 0x0c4fb5acUL, 0x4d7eaeb5UL, 0x8e2d839eUL,
0xcf1c9887UL, 0x5112c24aUL, 0x1023d953UL, 0xd370f478UL, 0x9241ef61UL,
0x55d7ae2eUL, 0x14e6b537UL, 0xd7b5981cUL, 0x96848305UL, 0x59981b82UL,
0x18a9009bUL, 0xdbfa2db0UL, 0x9acb36a9UL, 0x5d5d77e6UL, 0x1c6c6cffUL,
0xdf3f41d4UL, 0x9e0e5acdUL, 0xa2248495UL, 0xe3159f8cUL, 0x2046b2a7UL,
0x6177a9beUL, 0xa6e1e8f1UL, 0xe7d0f3e8UL, 0x2483dec3UL, 0x65b2c5daUL,
0xaaae5d5dUL, 0xeb9f4644UL, 0x28cc6b6fUL, 0x69fd7076UL, 0xae6b3139UL,
0xef5a2a20UL, 0x2c09070bUL, 0x6d381c12UL, 0xf33646dfUL, 0xb2075dc6UL,
0x715470edUL, 0x30656bf4UL, 0xf7f32abbUL, 0xb6c231a2UL, 0x75911c89UL,
0x34a00790UL, 0xfbbc9f17UL, 0xba8d840eUL, 0x79dea925UL, 0x38efb23cUL,
0xff79f373UL, 0xbe48e86aUL, 0x7d1bc541UL, 0x3c2ade58UL, 0x054f79f0UL,
0x447e62e9UL, 0x872d4fc2UL, 0xc61c54dbUL, 0x018a1594UL, 0x40bb0e8dUL,
0x83e823a6UL, 0xc2d938bfUL, 0x0dc5a038UL, 0x4cf4bb21UL, 0x8fa7960aUL,
0xce968d13UL, 0x0900cc5cUL, 0x4831d745UL, 0x8b62fa6eUL, 0xca53e177UL,
0x545dbbbaUL, 0x156ca0a3UL, 0xd63f8d88UL, 0x970e9691UL, 0x5098d7deUL,
0x11a9ccc7UL, 0xd2fae1ecUL, 0x93cbfaf5UL, 0x5cd76272UL, 0x1de6796bUL,
0xdeb55440UL, 0x9f844f59UL, 0x58120e16UL, 0x1923150fUL, 0xda703824UL,
0x9b41233dUL, 0xa76bfd65UL, 0xe65ae67cUL, 0x2509cb57UL, 0x6438d04eUL,
0xa3ae9101UL, 0xe29f8a18UL, 0x21cca733UL, 0x60fdbc2aUL, 0xafe124adUL,
0xeed03fb4UL, 0x2d83129fUL, 0x6cb20986UL, 0xab2448c9UL, 0xea1553d0UL,
0x29467efbUL, 0x687765e2UL, 0xf6793f2fUL, 0xb7482436UL, 0x741b091dUL,
0x352a1204UL, 0xf2bc534bUL, 0xb38d4852UL, 0x70de6579UL, 0x31ef7e60UL,
0xfef3e6e7UL, 0xbfc2fdfeUL, 0x7c91d0d5UL, 0x3da0cbccUL, 0xfa368a83UL,
0xbb07919aUL, 0x7854bcb1UL, 0x3965a7a8UL, 0x4b98833bUL, 0x0aa99822UL,
0xc9fab509UL, 0x88cbae10UL, 0x4f5def5fUL, 0x0e6cf446UL, 0xcd3fd96dUL,
0x8c0ec274UL, 0x43125af3UL, 0x022341eaUL, 0xc1706cc1UL, 0x804177d8UL,
0x47d73697UL, 0x06e62d8eUL, 0xc5b500a5UL, 0x84841bbcUL, 0x1a8a4171UL,
0x5bbb5a68UL, 0x98e87743UL, 0xd9d96c5aUL, 0x1e4f2d15UL, 0x5f7e360cUL,
0x9c2d1b27UL, 0xdd1c003eUL, 0x120098b9UL, 0x533183a0UL, 0x9062ae8bUL,
0xd153b592UL, 0x16c5f4ddUL, 0x57f4efc4UL, 0x94a7c2efUL, 0xd596d9f6UL,
0xe9bc07aeUL, 0xa88d1cb7UL, 0x6bde319cUL, 0x2aef2a85UL, 0xed796bcaUL,
0xac4870d3UL, 0x6f1b5df8UL, 0x2e2a46e1UL, 0xe136de66UL, 0xa007c57fUL,
0x6354e854UL, 0x2265f34dUL, 0xe5f3b202UL, 0xa4c2a91bUL, 0x67918430UL,
0x26a09f29UL, 0xb8aec5e4UL, 0xf99fdefdUL, 0x3accf3d6UL, 0x7bfde8cfUL,
0xbc6ba980UL, 0xfd5ab299UL, 0x3e099fb2UL, 0x7f3884abUL, 0xb0241c2cUL,
0xf1150735UL, 0x32462a1eUL, 0x73773107UL, 0xb4e17048UL, 0xf5d06b51UL,
0x3683467aUL, 0x77b25d63UL, 0x4ed7facbUL, 0x0fe6e1d2UL, 0xccb5ccf9UL,
0x8d84d7e0UL, 0x4a1296afUL, 0x0b238db6UL, 0xc870a09dUL, 0x8941bb84UL,
0x465d2303UL, 0x076c381aUL, 0xc43f1531UL, 0x850e0e28UL, 0x42984f67UL,
0x03a9547eUL, 0xc0fa7955UL, 0x81cb624cUL, 0x1fc53881UL, 0x5ef42398UL,
0x9da70eb3UL, 0xdc9615aaUL, 0x1b0054e5UL, 0x5a314ffcUL, 0x996262d7UL,
0xd85379ceUL, 0x174fe149UL, 0x567efa50UL, 0x952dd77bUL, 0xd41ccc62UL,
0x138a8d2dUL, 0x52bb9634UL, 0x91e8bb1fUL, 0xd0d9a006UL, 0xecf37e5eUL,
0xadc26547UL, 0x6e91486cUL, 0x2fa05375UL, 0xe836123aUL, 0xa9070923UL,
0x6a542408UL, 0x2b653f11UL, 0xe479a796UL, 0xa548bc8fUL, 0x661b91a4UL,
0x272a8abdUL, 0xe0bccbf2UL, 0xa18dd0ebUL, 0x62defdc0UL, 0x23efe6d9UL,
0xbde1bc14UL, 0xfcd0a70dUL, 0x3f838a26UL, 0x7eb2913fUL, 0xb924d070UL,
0xf815cb69UL, 0x3b46e642UL, 0x7a77fd5bUL, 0xb56b65dcUL, 0xf45a7ec5UL,
0x370953eeUL, 0x763848f7UL, 0xb1ae09b8UL, 0xf09f12a1UL, 0x33cc3f8aUL,
0x72fd2493UL
},
{
0x00000000UL, 0x376ac201UL, 0x6ed48403UL, 0x59be4602UL, 0xdca80907UL,
0xebc2cb06UL, 0xb27c8d04UL, 0x85164f05UL, 0xb851130eUL, 0x8f3bd10fUL,
0xd685970dUL, 0xe1ef550cUL, 0x64f91a09UL, 0x5393d808UL, 0x0a2d9e0aUL,
0x3d475c0bUL, 0x70a3261cUL, 0x47c9e41dUL, 0x1e77a21fUL, 0x291d601eUL,
0xac0b2f1bUL, 0x9b61ed1aUL, 0xc2dfab18UL, 0xf5b56919UL, 0xc8f23512UL,
0xff98f713UL, 0xa626b111UL, 0x914c7310UL, 0x145a3c15UL, 0x2330fe14UL,
0x7a8eb816UL, 0x4de47a17UL, 0xe0464d38UL, 0xd72c8f39UL, 0x8e92c93bUL,
0xb9f80b3aUL, 0x3cee443fUL, 0x0b84863eUL, 0x523ac03cUL, 0x6550023dUL,
0x58175e36UL, 0x6f7d9c37UL, 0x36c3da35UL, 0x01a91834UL, 0x84bf5731UL,
0xb3d59530UL, 0xea6bd332UL, 0xdd011133UL, 0x90e56b24UL, 0xa78fa925UL,
0xfe31ef27UL, 0xc95b2d26UL, 0x4c4d6223UL, 0x7b27a022UL, 0x2299e620UL,
0x15f32421UL, 0x28b4782aUL, 0x1fdeba2bUL, 0x4660fc29UL, 0x710a3e28UL,
0xf41c712dUL, 0xc376b32cUL, 0x9ac8f52eUL, 0xada2372fUL, 0xc08d9a70UL,
0xf7e75871UL, 0xae591e73UL, 0x9933dc72UL, 0x1c259377UL, 0x2b4f5176UL,
0x72f11774UL, 0x459bd575UL, 0x78dc897eUL, 0x4fb64b7fUL, 0x16080d7dUL,
0x2162cf7cUL, 0xa4748079UL, 0x931e4278UL, 0xcaa0047aUL, 0xfdcac67bUL,
0xb02ebc6cUL, 0x87447e6dUL, 0xdefa386fUL, 0xe990fa6eUL, 0x6c86b56bUL,
0x5bec776aUL, 0x02523168UL, 0x3538f369UL, 0x087faf62UL, 0x3f156d63UL,
0x66ab2b61UL, 0x51c1e960UL, 0xd4d7a665UL, 0xe3bd6464UL, 0xba032266UL,
0x8d69e067UL, 0x20cbd748UL, 0x17a11549UL, 0x4e1f534bUL, 0x7975914aUL,
0xfc63de4fUL, 0xcb091c4eUL, 0x92b75a4cUL, 0xa5dd984dUL, 0x989ac446UL,
0xaff00647UL, 0xf64e4045UL, 0xc1248244UL, 0x4432cd41UL, 0x73580f40UL,
0x2ae64942UL, 0x1d8c8b43UL, 0x5068f154UL, 0x67023355UL, 0x3ebc7557UL,
0x09d6b756UL, 0x8cc0f853UL, 0xbbaa3a52UL, 0xe2147c50UL, 0xd57ebe51UL,
0xe839e25aUL, 0xdf53205bUL, 0x86ed6659UL, 0xb187a458UL, 0x3491eb5dUL,
0x03fb295cUL, 0x5a456f5eUL, 0x6d2fad5fUL, 0x801b35e1UL, 0xb771f7e0UL,
0xeecfb1e2UL, 0xd9a573e3UL, 0x5cb33ce6UL, 0x6bd9fee7UL, 0x3267b8e5UL,
0x050d7ae4UL, 0x384a26efUL, 0x0f20e4eeUL, 0x569ea2ecUL, 0x61f460edUL,
0xe4e22fe8UL, 0xd388ede9UL, 0x8a36abebUL, 0xbd5c69eaUL, 0xf0b813fdUL,
0xc7d2d1fcUL, 0x9e6c97feUL, 0xa90655ffUL, 0x2c101afaUL, 0x1b7ad8fbUL,
0x42c49ef9UL, 0x75ae5cf8UL, 0x48e900f3UL, 0x7f83c2f2UL, 0x263d84f0UL,
0x115746f1UL, 0x944109f4UL, 0xa32bcbf5UL, 0xfa958df7UL, 0xcdff4ff6UL,
0x605d78d9UL, 0x5737bad8UL, 0x0e89fcdaUL, 0x39e33edbUL, 0xbcf571deUL,
0x8b9fb3dfUL, 0xd221f5ddUL, 0xe54b37dcUL, 0xd80c6bd7UL, 0xef66a9d6UL,
0xb6d8efd4UL, 0x81b22dd5UL, 0x04a462d0UL, 0x33cea0d1UL, 0x6a70e6d3UL,
0x5d1a24d2UL, 0x10fe5ec5UL, 0x27949cc4UL, 0x7e2adac6UL, 0x494018c7UL,
0xcc5657c2UL, 0xfb3c95c3UL, 0xa282d3c1UL, 0x95e811c0UL, 0xa8af4dcbUL,
0x9fc58fcaUL, 0xc67bc9c8UL, 0xf1110bc9UL, 0x740744ccUL, 0x436d86cdUL,
0x1ad3c0cfUL, 0x2db902ceUL, 0x4096af91UL, 0x77fc6d90UL, 0x2e422b92UL,
0x1928e993UL, 0x9c3ea696UL, 0xab546497UL, 0xf2ea2295UL, 0xc580e094UL,
0xf8c7bc9fUL, 0xcfad7e9eUL, 0x9613389cUL, 0xa179fa9dUL, 0x246fb598UL,
0x13057799UL, 0x4abb319bUL, 0x7dd1f39aUL, 0x3035898dUL, 0x075f4b8cUL,
0x5ee10d8eUL, 0x698bcf8fUL, 0xec9d808aUL, 0xdbf7428bUL, 0x82490489UL,
0xb523c688UL, 0x88649a83UL, 0xbf0e5882UL, 0xe6b01e80UL, 0xd1dadc81UL,
0x54cc9384UL, 0x63a65185UL, 0x3a181787UL, 0x0d72d586UL, 0xa0d0e2a9UL,
0x97ba20a8UL, 0xce0466aaUL, 0xf96ea4abUL, 0x7c78ebaeUL, 0x4b1229afUL,
0x12ac6fadUL, 0x25c6adacUL, 0x1881f1a7UL, 0x2feb33a6UL, 0x765575a4UL,
0x413fb7a5UL, 0xc429f8a0UL, 0xf3433aa1UL, 0xaafd7ca3UL, 0x9d97bea2UL,
0xd073c4b5UL, 0xe71906b4UL, 0xbea740b6UL, 0x89cd82b7UL, 0x0cdbcdb2UL,
0x3bb10fb3UL, 0x620f49b1UL, 0x55658bb0UL, 0x6822d7bbUL, 0x5f4815baUL,
0x06f653b8UL, 0x319c91b9UL, 0xb48adebcUL, 0x83e01cbdUL, 0xda5e5abfUL,
0xed3498beUL
},
{
0x00000000UL, 0x6567bcb8UL, 0x8bc809aaUL, 0xeeafb512UL, 0x5797628fUL,
0x32f0de37UL, 0xdc5f6b25UL, 0xb938d79dUL, 0xef28b4c5UL, 0x8a4f087dUL,
0x64e0bd6fUL, 0x018701d7UL, 0xb8bfd64aUL, 0xddd86af2UL, 0x3377dfe0UL,
0x56106358UL, 0x9f571950UL, 0xfa30a5e8UL, 0x149f10faUL, 0x71f8ac42UL,
0xc8c07bdfUL, 0xada7c767UL, 0x43087275UL, 0x266fcecdUL, 0x707fad95UL,
0x1518112dUL, 0xfbb7a43fUL, 0x9ed01887UL, 0x27e8cf1aUL, 0x428f73a2UL,
0xac20c6b0UL, 0xc9477a08UL, 0x3eaf32a0UL, 0x5bc88e18UL, 0xb5673b0aUL,
0xd00087b2UL, 0x6938502fUL, 0x0c5fec97UL, 0xe2f05985UL, 0x8797e53dUL,
0xd1878665UL, 0xb4e03addUL, 0x5a4f8fcfUL, 0x3f283377UL, 0x8610e4eaUL,
0xe3775852UL, 0x0dd8ed40UL, 0x68bf51f8UL, 0xa1f82bf0UL, 0xc49f9748UL,
0x2a30225aUL, 0x4f579ee2UL, 0xf66f497fUL, 0x9308f5c7UL, 0x7da740d5UL,
0x18c0fc6dUL, 0x4ed09f35UL, 0x2bb7238dUL, 0xc518969fUL, 0xa07f2a27UL,
0x1947fdbaUL, 0x7c204102UL, 0x928ff410UL, 0xf7e848a8UL, 0x3d58149bUL,
0x583fa823UL, 0xb6901d31UL, 0xd3f7a189UL, 0x6acf7614UL, 0x0fa8caacUL,
0xe1077fbeUL, 0x8460c306UL, 0xd270a05eUL, 0xb7171ce6UL, 0x59b8a9f4UL,
0x3cdf154cUL, 0x85e7c2d1UL, 0xe0807e69UL, 0x0e2fcb7bUL, 0x6b4877c3UL,
0xa20f0dcbUL, 0xc768b173UL, 0x29c70461UL, 0x4ca0b8d9UL, 0xf5986f44UL,
0x90ffd3fcUL, 0x7e5066eeUL, 0x1b37da56UL, 0x4d27b90eUL, 0x284005b6UL,
0xc6efb0a4UL, 0xa3880c1cUL, 0x1ab0db81UL, 0x7fd76739UL, 0x9178d22bUL,
0xf41f6e93UL, 0x03f7263bUL, 0x66909a83UL, 0x883f2f91UL, 0xed589329UL,
0x546044b4UL, 0x3107f80cUL, 0xdfa84d1eUL, 0xbacff1a6UL, 0xecdf92feUL,
0x89b82e46UL, 0x67179b54UL, 0x027027ecUL, 0xbb48f071UL, 0xde2f4cc9UL,
0x3080f9dbUL, 0x55e74563UL, 0x9ca03f6bUL, 0xf9c783d3UL, 0x176836c1UL,
0x720f8a79UL, 0xcb375de4UL, 0xae50e15cUL, 0x40ff544eUL, 0x2598e8f6UL,
0x73888baeUL, 0x16ef3716UL, 0xf8408204UL, 0x9d273ebcUL, 0x241fe921UL,
0x41785599UL, 0xafd7e08bUL, 0xcab05c33UL, 0x3bb659edUL, 0x5ed1e555UL,
0xb07e5047UL, 0xd519ecffUL, 0x6c213b62UL, 0x094687daUL, 0xe7e932c8UL,
0x828e8e70UL, 0xd49eed28UL, 0xb1f95190UL, 0x5f56e482UL, 0x3a31583aUL,
0x83098fa7UL, 0xe66e331fUL, 0x08c1860dUL, 0x6da63ab5UL, 0xa4e140bdUL,
0xc186fc05UL, 0x2f294917UL, 0x4a4ef5afUL, 0xf3762232UL, 0x96119e8aUL,
0x78be2b98UL, 0x1dd99720UL, 0x4bc9f478UL, 0x2eae48c0UL, 0xc001fdd2UL,
0xa566416aUL, 0x1c5e96f7UL, 0x79392a4fUL, 0x97969f5dUL, 0xf2f123e5UL,
0x05196b4dUL, 0x607ed7f5UL, 0x8ed162e7UL, 0xebb6de5fUL, 0x528e09c2UL,
0x37e9b57aUL, 0xd9460068UL, 0xbc21bcd0UL, 0xea31df88UL, 0x8f566330UL,
0x61f9d622UL, 0x049e6a9aUL, 0xbda6bd07UL, 0xd8c101bfUL, 0x366eb4adUL,
0x53090815UL, 0x9a4e721dUL, 0xff29cea5UL, 0x11867bb7UL, 0x74e1c70fUL,
0xcdd91092UL, 0xa8beac2aUL, 0x46111938UL, 0x2376a580UL, 0x7566c6d8UL,
0x10017a60UL, 0xfeaecf72UL, 0x9bc973caUL, 0x22f1a457UL, 0x479618efUL,
0xa939adfdUL, 0xcc5e1145UL, 0x06ee4d76UL, 0x6389f1ceUL, 0x8d2644dcUL,
0xe841f864UL, 0x51792ff9UL, 0x341e9341UL, 0xdab12653UL, 0xbfd69aebUL,
0xe9c6f9b3UL, 0x8ca1450bUL, 0x620ef019UL, 0x07694ca1UL, 0xbe519b3cUL,
0xdb362784UL, 0x35999296UL, 0x50fe2e2eUL, 0x99b95426UL, 0xfcdee89eUL,
0x12715d8cUL, 0x7716e134UL, 0xce2e36a9UL, 0xab498a11UL, 0x45e63f03UL,
0x208183bbUL, 0x7691e0e3UL, 0x13f65c5bUL, 0xfd59e949UL, 0x983e55f1UL,
0x2106826cUL, 0x44613ed4UL, 0xaace8bc6UL, 0xcfa9377eUL, 0x38417fd6UL,
0x5d26c36eUL, 0xb389767cUL, 0xd6eecac4UL, 0x6fd61d59UL, 0x0ab1a1e1UL,
0xe41e14f3UL, 0x8179a84bUL, 0xd769cb13UL, 0xb20e77abUL, 0x5ca1c2b9UL,
0x39c67e01UL, 0x80fea99cUL, 0xe5991524UL, 0x0b36a036UL, 0x6e511c8eUL,
0xa7166686UL, 0xc271da3eUL, 0x2cde6f2cUL, 0x49b9d394UL, 0xf0810409UL,
0x95e6b8b1UL, 0x7b490da3UL, 0x1e2eb11bUL, 0x483ed243UL, 0x2d596efbUL,
0xc3f6dbe9UL, 0xa6916751UL, 0x1fa9b0ccUL, 0x7ace0c74UL, 0x9461b966UL,
0xf10605deUL
#endif
}
};

1967
fdbrpc/zlib/deflate.c
View File

File diff suppressed because it is too large Load Diff

346
fdbrpc/zlib/deflate.h
View File

@ -1,346 +0,0 @@
/* deflate.h -- internal compression state
* Copyright (C) 1995-2012 Jean-loup Gailly
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* WARNING: this file should *not* be used by applications. It is
part of the implementation of the compression library and is
subject to change. Applications should only use zlib.h.
*/
/* @(#) $Id$ */
#ifndef DEFLATE_H
#define DEFLATE_H
#include "zutil.h"
/* define NO_GZIP when compiling if you want to disable gzip header and
trailer creation by deflate(). NO_GZIP would be used to avoid linking in
the crc code when it is not needed. For shared libraries, gzip encoding
should be left enabled. */
#ifndef NO_GZIP
# define GZIP
#endif
/* ===========================================================================
* Internal compression state.
*/
#define LENGTH_CODES 29
/* number of length codes, not counting the special END_BLOCK code */
#define LITERALS 256
/* number of literal bytes 0..255 */
#define L_CODES (LITERALS+1+LENGTH_CODES)
/* number of Literal or Length codes, including the END_BLOCK code */
#define D_CODES 30
/* number of distance codes */
#define BL_CODES 19
/* number of codes used to transfer the bit lengths */
#define HEAP_SIZE (2*L_CODES+1)
/* maximum heap size */
#define MAX_BITS 15
/* All codes must not exceed MAX_BITS bits */
#define Buf_size 16
/* size of bit buffer in bi_buf */
#define INIT_STATE 42
#define EXTRA_STATE 69
#define NAME_STATE 73
#define COMMENT_STATE 91
#define HCRC_STATE 103
#define BUSY_STATE 113
#define FINISH_STATE 666
/* Stream status */
/* Data structure describing a single value and its code string. */
typedef struct ct_data_s {
union {
ush freq; /* frequency count */
ush code; /* bit string */
} fc;
union {
ush dad; /* father node in Huffman tree */
ush len; /* length of bit string */
} dl;
} FAR ct_data;
#define Freq fc.freq
#define Code fc.code
#define Dad dl.dad
#define Len dl.len
typedef struct static_tree_desc_s static_tree_desc;
typedef struct tree_desc_s {
ct_data *dyn_tree; /* the dynamic tree */
int max_code; /* largest code with non zero frequency */
static_tree_desc *stat_desc; /* the corresponding static tree */
} FAR tree_desc;
typedef ush Pos;
typedef Pos FAR Posf;
typedef unsigned IPos;
/* A Pos is an index in the character window. We use short instead of int to
* save space in the various tables. IPos is used only for parameter passing.
*/
typedef struct internal_state {
z_streamp strm; /* pointer back to this zlib stream */
int status; /* as the name implies */
Bytef *pending_buf; /* output still pending */
ulg pending_buf_size; /* size of pending_buf */
Bytef *pending_out; /* next pending byte to output to the stream */
uInt pending; /* nb of bytes in the pending buffer */
int wrap; /* bit 0 true for zlib, bit 1 true for gzip */
gz_headerp gzhead; /* gzip header information to write */
uInt gzindex; /* where in extra, name, or comment */
Byte method; /* can only be DEFLATED */
int last_flush; /* value of flush param for previous deflate call */
/* used by deflate.c: */
uInt w_size; /* LZ77 window size (32K by default) */
uInt w_bits; /* log2(w_size) (8..16) */
uInt w_mask; /* w_size - 1 */
Bytef *window;
/* Sliding window. Input bytes are read into the second half of the window,
* and move to the first half later to keep a dictionary of at least wSize
* bytes. With this organization, matches are limited to a distance of
* wSize-MAX_MATCH bytes, but this ensures that IO is always
* performed with a length multiple of the block size. Also, it limits
* the window size to 64K, which is quite useful on MSDOS.
* To do: use the user input buffer as sliding window.
*/
ulg window_size;
/* Actual size of window: 2*wSize, except when the user input buffer
* is directly used as sliding window.
*/
Posf *prev;
/* Link to older string with same hash index. To limit the size of this
* array to 64K, this link is maintained only for the last 32K strings.
* An index in this array is thus a window index modulo 32K.
*/
Posf *head; /* Heads of the hash chains or NIL. */
uInt ins_h; /* hash index of string to be inserted */
uInt hash_size; /* number of elements in hash table */
uInt hash_bits; /* log2(hash_size) */
uInt hash_mask; /* hash_size-1 */
uInt hash_shift;
/* Number of bits by which ins_h must be shifted at each input
* step. It must be such that after MIN_MATCH steps, the oldest
* byte no longer takes part in the hash key, that is:
* hash_shift * MIN_MATCH >= hash_bits
*/
long block_start;
/* Window position at the beginning of the current output block. Gets
* negative when the window is moved backwards.
*/
uInt match_length; /* length of best match */
IPos prev_match; /* previous match */
int match_available; /* set if previous match exists */
uInt strstart; /* start of string to insert */
uInt match_start; /* start of matching string */
uInt lookahead; /* number of valid bytes ahead in window */
uInt prev_length;
/* Length of the best match at previous step. Matches not greater than this
* are discarded. This is used in the lazy match evaluation.
*/
uInt max_chain_length;
/* To speed up deflation, hash chains are never searched beyond this
* length. A higher limit improves compression ratio but degrades the
* speed.
*/
uInt max_lazy_match;
/* Attempt to find a better match only when the current match is strictly
* smaller than this value. This mechanism is used only for compression
* levels >= 4.
*/
# define max_insert_length max_lazy_match
/* Insert new strings in the hash table only if the match length is not
* greater than this length. This saves time but degrades compression.
* max_insert_length is used only for compression levels <= 3.
*/
int level; /* compression level (1..9) */
int strategy; /* favor or force Huffman coding*/
uInt good_match;
/* Use a faster search when the previous match is longer than this */
int nice_match; /* Stop searching when current match exceeds this */
/* used by trees.c: */
/* Didn't use ct_data typedef below to suppress compiler warning */
struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */
struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */
struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */
struct tree_desc_s l_desc; /* desc. for literal tree */
struct tree_desc_s d_desc; /* desc. for distance tree */
struct tree_desc_s bl_desc; /* desc. for bit length tree */
ush bl_count[MAX_BITS+1];
/* number of codes at each bit length for an optimal tree */
int heap[2*L_CODES+1]; /* heap used to build the Huffman trees */
int heap_len; /* number of elements in the heap */
int heap_max; /* element of largest frequency */
/* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used.
* The same heap array is used to build all trees.
*/
uch depth[2*L_CODES+1];
/* Depth of each subtree used as tie breaker for trees of equal frequency
*/
uchf *l_buf; /* buffer for literals or lengths */
uInt lit_bufsize;
/* Size of match buffer for literals/lengths. There are 4 reasons for
* limiting lit_bufsize to 64K:
* - frequencies can be kept in 16 bit counters
* - if compression is not successful for the first block, all input
* data is still in the window so we can still emit a stored block even
* when input comes from standard input. (This can also be done for
* all blocks if lit_bufsize is not greater than 32K.)
* - if compression is not successful for a file smaller than 64K, we can
* even emit a stored file instead of a stored block (saving 5 bytes).
* This is applicable only for zip (not gzip or zlib).
* - creating new Huffman trees less frequently may not provide fast
* adaptation to changes in the input data statistics. (Take for
* example a binary file with poorly compressible code followed by
* a highly compressible string table.) Smaller buffer sizes give
* fast adaptation but have of course the overhead of transmitting
* trees more frequently.
* - I can't count above 4
*/
uInt last_lit; /* running index in l_buf */
ushf *d_buf;
/* Buffer for distances. To simplify the code, d_buf and l_buf have
* the same number of elements. To use different lengths, an extra flag
* array would be necessary.
*/
ulg opt_len; /* bit length of current block with optimal trees */
ulg static_len; /* bit length of current block with static trees */
uInt matches; /* number of string matches in current block */
uInt insert; /* bytes at end of window left to insert */
#ifdef DEBUG
ulg compressed_len; /* total bit length of compressed file mod 2^32 */
ulg bits_sent; /* bit length of compressed data sent mod 2^32 */
#endif
ush bi_buf;
/* Output buffer. bits are inserted starting at the bottom (least
* significant bits).
*/
int bi_valid;
/* Number of valid bits in bi_buf. All bits above the last valid bit
* are always zero.
*/
ulg high_water;
/* High water mark offset in window for initialized bytes -- bytes above
* this are set to zero in order to avoid memory check warnings when
* longest match routines access bytes past the input. This is then
* updated to the new high water mark.
*/
} FAR deflate_state;
/* Output a byte on the stream.
* IN assertion: there is enough room in pending_buf.
*/
#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);}
#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1)
/* Minimum amount of lookahead, except at the end of the input file.
* See deflate.c for comments about the MIN_MATCH+1.
*/
#define MAX_DIST(s) ((s)->w_size-MIN_LOOKAHEAD)
/* In order to simplify the code, particularly on 16 bit machines, match
* distances are limited to MAX_DIST instead of WSIZE.
*/
#define WIN_INIT MAX_MATCH
/* Number of bytes after end of data in window to initialize in order to avoid
memory checker errors from longest match routines */
/* in trees.c */
void ZLIB_INTERNAL _tr_init OF((deflate_state *s));
int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc));
void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf,
ulg stored_len, int last));
void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s));
void ZLIB_INTERNAL _tr_align OF((deflate_state *s));
void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
ulg stored_len, int last));
#define d_code(dist) \
((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)])
/* Mapping from a distance to a distance code. dist is the distance - 1 and
* must not have side effects. _dist_code[256] and _dist_code[257] are never
* used.
*/
#ifndef DEBUG
/* Inline versions of _tr_tally for speed: */
#if defined(GEN_TREES_H) || !defined(STDC)
extern uch ZLIB_INTERNAL _length_code[];
extern uch ZLIB_INTERNAL _dist_code[];
#else
extern const uch ZLIB_INTERNAL _length_code[];
extern const uch ZLIB_INTERNAL _dist_code[];
#endif
# define _tr_tally_lit(s, c, flush) \
{ uch cc = (c); \
s->d_buf[s->last_lit] = 0; \
s->l_buf[s->last_lit++] = cc; \
s->dyn_ltree[cc].Freq++; \
flush = (s->last_lit == s->lit_bufsize-1); \
}
# define _tr_tally_dist(s, distance, length, flush) \
{ uch len = (length); \
ush dist = (distance); \
s->d_buf[s->last_lit] = dist; \
s->l_buf[s->last_lit++] = len; \
dist--; \
s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
s->dyn_dtree[d_code(dist)].Freq++; \
flush = (s->last_lit == s->lit_bufsize-1); \
}
#else
# define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c)
# define _tr_tally_dist(s, distance, length, flush) \
flush = _tr_tally(s, distance, length)
#endif
#endif /* DEFLATE_H */

25
fdbrpc/zlib/gzclose.c
View File

@ -1,25 +0,0 @@
/* gzclose.c -- zlib gzclose() function
* Copyright (C) 2004, 2010 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "gzguts.h"
/* gzclose() is in a separate file so that it is linked in only if it is used.
That way the other gzclose functions can be used instead to avoid linking in
unneeded compression or decompression routines. */
int ZEXPORT gzclose(file)
gzFile file;
{
#ifndef NO_GZCOMPRESS
gz_statep state;
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
return state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file);
#else
return gzclose_r(file);
#endif
}

211
fdbrpc/zlib/gzguts.h
View File

@ -1,211 +0,0 @@
/* gzguts.h -- zlib internal header definitions for gz* operations
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#ifdef _LARGEFILE64_SOURCE
# ifndef _LARGEFILE_SOURCE
# define _LARGEFILE_SOURCE 1
# endif
# ifdef _FILE_OFFSET_BITS
# undef _FILE_OFFSET_BITS
# endif
#endif
#ifdef HAVE_HIDDEN
# define ZLIB_INTERNAL __attribute__((visibility ("hidden")))
#else
# define ZLIB_INTERNAL
#endif
#include <stdio.h>
#include "zlib.h"
#ifdef STDC
# include <string.h>
# include <stdlib.h>
# include <limits.h>
#endif
#include <fcntl.h>
#ifdef _WIN32
#include <stddef.h>
#else
#include <unistd.h>
#endif
#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32)
# include <io.h>
#endif
#ifdef WINAPI_FAMILY
# define open _open
# define read _read
# define write _write
# define close _close
#endif
#ifdef NO_DEFLATE /* for compatibility with old definition */
# define NO_GZCOMPRESS
#endif
#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550)
# ifndef HAVE_VSNPRINTF
# define HAVE_VSNPRINTF
# endif
#endif
#if defined(__CYGWIN__)
# ifndef HAVE_VSNPRINTF
# define HAVE_VSNPRINTF
# endif
#endif
#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410)
# ifndef HAVE_VSNPRINTF
# define HAVE_VSNPRINTF
# endif
#endif
#ifndef HAVE_VSNPRINTF
# ifdef MSDOS
/* vsnprintf may exist on some MS-DOS compilers (DJGPP?),
but for now we just assume it doesn't. */
# define NO_vsnprintf
# endif
# ifdef __TURBOC__
# define NO_vsnprintf
# endif
# ifdef WIN32
/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */
# if !defined(vsnprintf) && !defined(NO_vsnprintf)
# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 )
# define vsnprintf _vsnprintf
# endif
# endif
# endif
# ifdef __SASC
# define NO_vsnprintf
# endif
# ifdef VMS
# define NO_vsnprintf
# endif
# ifdef __OS400__
# define NO_vsnprintf
# endif
# ifdef __MVS__
# define NO_vsnprintf
# endif
#endif
/* unlike snprintf (which is required in C99, yet still not supported by
Microsoft more than a decade later!), _snprintf does not guarantee null
termination of the result -- however this is only used in gzlib.c where
the result is assured to fit in the space provided */
#ifdef _MSC_VER
# define snprintf _snprintf
#endif
#ifndef local
# define local static
#endif
/* compile with -Dlocal if your debugger can't find static symbols */
/* gz* functions always use library allocation functions */
#ifndef STDC
extern voidp malloc OF((uInt size));
extern void free OF((voidpf ptr));
#endif
/* get errno and strerror definition */
#if defined UNDER_CE
# include <windows.h>
# define zstrerror() gz_strwinerror((DWORD)GetLastError())
#else
# ifndef NO_STRERROR
# include <errno.h>
# define zstrerror() strerror(errno)
# else
# define zstrerror() "stdio error (consult errno)"
# endif
#endif
/* provide prototypes for these when building zlib without LFS */
#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0
ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
#endif
/* default memLevel */
#if MAX_MEM_LEVEL >= 8
# define DEF_MEM_LEVEL 8
#else
# define DEF_MEM_LEVEL MAX_MEM_LEVEL
#endif
/* default i/o buffer size -- double this for output when reading (this and
twice this must be able to fit in an unsigned type) */
#define GZBUFSIZE 8192
/* gzip modes, also provide a little integrity check on the passed structure */
#define GZ_NONE 0
#define GZ_READ 7247
#define GZ_WRITE 31153
#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */
/* values for gz_state how */
#define LOOK 0 /* look for a gzip header */
#define COPY 1 /* copy input directly */
#define GZIP 2 /* decompress a gzip stream */
/* internal gzip file state data structure */
typedef struct {
/* exposed contents for gzgetc() macro */
struct gzFile_s x; /* "x" for exposed */
/* x.have: number of bytes available at x.next */
/* x.next: next output data to deliver or write */
/* x.pos: current position in uncompressed data */
/* used for both reading and writing */
int mode; /* see gzip modes above */
int fd; /* file descriptor */
char *path; /* path or fd for error messages */
unsigned size; /* buffer size, zero if not allocated yet */
unsigned want; /* requested buffer size, default is GZBUFSIZE */
unsigned char *in; /* input buffer */
unsigned char *out; /* output buffer (double-sized when reading) */
int direct; /* 0 if processing gzip, 1 if transparent */
/* just for reading */
int how; /* 0: get header, 1: copy, 2: decompress */
z_off64_t start; /* where the gzip data started, for rewinding */
int eof; /* true if end of input file reached */
int past; /* true if read requested past end */
/* just for writing */
int level; /* compression level */
int strategy; /* compression strategy */
/* seek request */
z_off64_t skip; /* amount to skip (already rewound if backwards) */
int seek; /* true if seek request pending */
/* error information */
int err; /* error code */
char *msg; /* error message */
/* zlib inflate or deflate stream */
z_stream strm; /* stream structure in-place (not a pointer) */
} gz_state;
typedef gz_state FAR *gz_statep;
/* shared functions */
void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *));
#if defined UNDER_CE
char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error));
#endif
/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t
value -- needed when comparing unsigned to z_off64_t, which is signed
(possible z_off64_t types off_t, off64_t, and long are all signed) */
#ifdef INT_MAX
# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX)
#else
unsigned ZLIB_INTERNAL gz_intmax OF((void));
# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax())
#endif

634
fdbrpc/zlib/gzlib.c
View File

@ -1,634 +0,0 @@
/* gzlib.c -- zlib functions common to reading and writing gzip files
* Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "gzguts.h"
#if defined(_WIN32) && !defined(__BORLANDC__)
# define LSEEK _lseeki64
#else
#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0
# define LSEEK lseek64
#else
# define LSEEK lseek
#endif
#endif
/* Local functions */
local void gz_reset OF((gz_statep));
local gzFile gz_open OF((const void *, int, const char *));
#if defined UNDER_CE
/* Map the Windows error number in ERROR to a locale-dependent error message
string and return a pointer to it. Typically, the values for ERROR come
from GetLastError.
The string pointed to shall not be modified by the application, but may be
overwritten by a subsequent call to gz_strwinerror
The gz_strwinerror function does not change the current setting of
GetLastError. */
char ZLIB_INTERNAL *gz_strwinerror (error)
DWORD error;
{
static char buf[1024];
wchar_t *msgbuf;
DWORD lasterr = GetLastError();
DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM
| FORMAT_MESSAGE_ALLOCATE_BUFFER,
NULL,
error,
0, /* Default language */
(LPVOID)&msgbuf,
0,
NULL);
if (chars != 0) {
/* If there is an \r\n appended, zap it. */
if (chars >= 2
&& msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') {
chars -= 2;
msgbuf[chars] = 0;
}
if (chars > sizeof (buf) - 1) {
chars = sizeof (buf) - 1;
msgbuf[chars] = 0;
}
wcstombs(buf, msgbuf, chars + 1);
LocalFree(msgbuf);
}
else {
sprintf(buf, "unknown win32 error (%ld)", error);
}
SetLastError(lasterr);
return buf;
}
#endif /* UNDER_CE */
/* Reset gzip file state */
local void gz_reset(state)
gz_statep state;
{
state->x.have = 0; /* no output data available */
if (state->mode == GZ_READ) { /* for reading ... */
state->eof = 0; /* not at end of file */
state->past = 0; /* have not read past end yet */
state->how = LOOK; /* look for gzip header */
}
state->seek = 0; /* no seek request pending */
gz_error(state, Z_OK, NULL); /* clear error */
state->x.pos = 0; /* no uncompressed data yet */
state->strm.avail_in = 0; /* no input data yet */
}
/* Open a gzip file either by name or file descriptor. */
local gzFile gz_open(path, fd, mode)
const void *path;
int fd;
const char *mode;
{
gz_statep state;
size_t len;
int oflag;
#ifdef O_CLOEXEC
int cloexec = 0;
#endif
#ifdef O_EXCL
int exclusive = 0;
#endif
/* check input */
if (path == NULL)
return NULL;
/* allocate gzFile structure to return */
state = (gz_statep)malloc(sizeof(gz_state));
if (state == NULL)
return NULL;
state->size = 0; /* no buffers allocated yet */
state->want = GZBUFSIZE; /* requested buffer size */
state->msg = NULL; /* no error message yet */
/* interpret mode */
state->mode = GZ_NONE;
state->level = Z_DEFAULT_COMPRESSION;
state->strategy = Z_DEFAULT_STRATEGY;
state->direct = 0;
while (*mode) {
if (*mode >= '0' && *mode <= '9')
state->level = *mode - '0';
else
switch (*mode) {
case 'r':
state->mode = GZ_READ;
break;
#ifndef NO_GZCOMPRESS
case 'w':
state->mode = GZ_WRITE;
break;
case 'a':
state->mode = GZ_APPEND;
break;
#endif
case '+': /* can't read and write at the same time */
free(state);
return NULL;
case 'b': /* ignore -- will request binary anyway */
break;
#ifdef O_CLOEXEC
case 'e':
cloexec = 1;
break;
#endif
#ifdef O_EXCL
case 'x':
exclusive = 1;
break;
#endif
case 'f':
state->strategy = Z_FILTERED;
break;
case 'h':
state->strategy = Z_HUFFMAN_ONLY;
break;
case 'R':
state->strategy = Z_RLE;
break;
case 'F':
state->strategy = Z_FIXED;
break;
case 'T':
state->direct = 1;
break;
default: /* could consider as an error, but just ignore */
;
}
mode++;
}
/* must provide an "r", "w", or "a" */
if (state->mode == GZ_NONE) {
free(state);
return NULL;
}
/* can't force transparent read */
if (state->mode == GZ_READ) {
if (state->direct) {
free(state);
return NULL;
}
state->direct = 1; /* for empty file */
}
/* save the path name for error messages */
#ifdef _WIN32
if (fd == -2) {
len = wcstombs(NULL, path, 0);
if (len == (size_t)-1)
len = 0;
}
else
#endif
len = strlen((const char *)path);
state->path = (char *)malloc(len + 1);
if (state->path == NULL) {
free(state);
return NULL;
}
#ifdef _WIN32
if (fd == -2)
if (len)
wcstombs(state->path, path, len + 1);
else
*(state->path) = 0;
else
#endif
#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
snprintf(state->path, len + 1, "%s", (const char *)path);
#else
strcpy(state->path, path);
#endif
/* compute the flags for open() */
oflag =
#ifdef O_LARGEFILE
O_LARGEFILE |
#endif
#ifdef O_BINARY
O_BINARY |
#endif
#ifdef O_CLOEXEC
(cloexec ? O_CLOEXEC : 0) |
#endif
(state->mode == GZ_READ ?
O_RDONLY :
(O_WRONLY | O_CREAT |
#ifdef O_EXCL
(exclusive ? O_EXCL : 0) |
#endif
(state->mode == GZ_WRITE ?
O_TRUNC :
O_APPEND)));
/* open the file with the appropriate flags (or just use fd) */
state->fd = fd > -1 ? fd : (
#ifdef _WIN32
fd == -2 ? _wopen(path, oflag, 0666) :
#endif
open((const char *)path, oflag, 0666));
if (state->fd == -1) {
free(state->path);
free(state);
return NULL;
}
if (state->mode == GZ_APPEND)
state->mode = GZ_WRITE; /* simplify later checks */
/* save the current position for rewinding (only if reading) */
if (state->mode == GZ_READ) {
state->start = LSEEK(state->fd, 0, SEEK_CUR);
if (state->start == -1) state->start = 0;
}
/* initialize stream */
gz_reset(state);
/* return stream */
return (gzFile)state;
}
/* -- see zlib.h -- */
gzFile ZEXPORT gzopen(path, mode)
const char *path;
const char *mode;
{
return gz_open(path, -1, mode);
}
/* -- see zlib.h -- */
gzFile ZEXPORT gzopen64(path, mode)
const char *path;
const char *mode;
{
return gz_open(path, -1, mode);
}
/* -- see zlib.h -- */
gzFile ZEXPORT gzdopen(fd, mode)
int fd;
const char *mode;
{
char *path; /* identifier for error messages */
gzFile gz;
if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL)
return NULL;
#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
snprintf(path, 7 + 3 * sizeof(int), "<fd:%d>", fd); /* for debugging */
#else
sprintf(path, "<fd:%d>", fd); /* for debugging */
#endif
gz = gz_open(path, fd, mode);
free(path);
return gz;
}
/* -- see zlib.h -- */
#ifdef _WIN32
gzFile ZEXPORT gzopen_w(path, mode)
const wchar_t *path;
const char *mode;
{
return gz_open(path, -2, mode);
}
#endif
/* -- see zlib.h -- */
int ZEXPORT gzbuffer(file, size)
gzFile file;
unsigned size;
{
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return -1;
/* make sure we haven't already allocated memory */
if (state->size != 0)
return -1;
/* check and set requested size */
if (size < 2)
size = 2; /* need two bytes to check magic header */
state->want = size;
return 0;
}
/* -- see zlib.h -- */
int ZEXPORT gzrewind(file)
gzFile file;
{
gz_statep state;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
/* check that we're reading and that there's no error */
if (state->mode != GZ_READ ||
(state->err != Z_OK && state->err != Z_BUF_ERROR))
return -1;
/* back up and start over */
if (LSEEK(state->fd, state->start, SEEK_SET) == -1)
return -1;
gz_reset(state);
return 0;
}
/* -- see zlib.h -- */
z_off64_t ZEXPORT gzseek64(file, offset, whence)
gzFile file;
z_off64_t offset;
int whence;
{
unsigned n;
z_off64_t ret;
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return -1;
/* check that there's no error */
if (state->err != Z_OK && state->err != Z_BUF_ERROR)
return -1;
/* can only seek from start or relative to current position */
if (whence != SEEK_SET && whence != SEEK_CUR)
return -1;
/* normalize offset to a SEEK_CUR specification */
if (whence == SEEK_SET)
offset -= state->x.pos;
else if (state->seek)
offset += state->skip;
state->seek = 0;
/* if within raw area while reading, just go there */
if (state->mode == GZ_READ && state->how == COPY &&
state->x.pos + offset >= 0) {
ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR);
if (ret == -1)
return -1;
state->x.have = 0;
state->eof = 0;
state->past = 0;
state->seek = 0;
gz_error(state, Z_OK, NULL);
state->strm.avail_in = 0;
state->x.pos += offset;
return state->x.pos;
}
/* calculate skip amount, rewinding if needed for back seek when reading */
if (offset < 0) {
if (state->mode != GZ_READ) /* writing -- can't go backwards */
return -1;
offset += state->x.pos;
if (offset < 0) /* before start of file! */
return -1;
if (gzrewind(file) == -1) /* rewind, then skip to offset */
return -1;
}
/* if reading, skip what's in output buffer (one less gzgetc() check) */
if (state->mode == GZ_READ) {
n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ?
(unsigned)offset : state->x.have;
state->x.have -= n;
state->x.next += n;
state->x.pos += n;
offset -= n;
}
/* request skip (if not zero) */
if (offset) {
state->seek = 1;
state->skip = offset;
}
return state->x.pos + offset;
}
/* -- see zlib.h -- */
z_off_t ZEXPORT gzseek(file, offset, whence)
gzFile file;
z_off_t offset;
int whence;
{
z_off64_t ret;
ret = gzseek64(file, (z_off64_t)offset, whence);
return ret == (z_off_t)ret ? (z_off_t)ret : -1;
}
/* -- see zlib.h -- */
z_off64_t ZEXPORT gztell64(file)
gzFile file;
{
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return -1;
/* return position */
return state->x.pos + (state->seek ? state->skip : 0);
}
/* -- see zlib.h -- */
z_off_t ZEXPORT gztell(file)
gzFile file;
{
z_off64_t ret;
ret = gztell64(file);
return ret == (z_off_t)ret ? (z_off_t)ret : -1;
}
/* -- see zlib.h -- */
z_off64_t ZEXPORT gzoffset64(file)
gzFile file;
{
z_off64_t offset;
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return -1;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return -1;
/* compute and return effective offset in file */
offset = LSEEK(state->fd, 0, SEEK_CUR);
if (offset == -1)
return -1;
if (state->mode == GZ_READ) /* reading */
offset -= state->strm.avail_in; /* don't count buffered input */
return offset;
}
/* -- see zlib.h -- */
z_off_t ZEXPORT gzoffset(file)
gzFile file;
{
z_off64_t ret;
ret = gzoffset64(file);
return ret == (z_off_t)ret ? (z_off_t)ret : -1;
}
/* -- see zlib.h -- */
int ZEXPORT gzeof(file)
gzFile file;
{
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return 0;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return 0;
/* return end-of-file state */
return state->mode == GZ_READ ? state->past : 0;
}
/* -- see zlib.h -- */
const char * ZEXPORT gzerror(file, errnum)
gzFile file;
int *errnum;
{
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return NULL;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return NULL;
/* return error information */
if (errnum != NULL)
*errnum = state->err;
return state->err == Z_MEM_ERROR ? "out of memory" :
(state->msg == NULL ? "" : state->msg);
}
/* -- see zlib.h -- */
void ZEXPORT gzclearerr(file)
gzFile file;
{
gz_statep state;
/* get internal structure and check integrity */
if (file == NULL)
return;
state = (gz_statep)file;
if (state->mode != GZ_READ && state->mode != GZ_WRITE)
return;
/* clear error and end-of-file */
if (state->mode == GZ_READ) {
state->eof = 0;
state->past = 0;
}
gz_error(state, Z_OK, NULL);
}
/* Create an error message in allocated memory and set state->err and
state->msg accordingly. Free any previous error message already there. Do
not try to free or allocate space if the error is Z_MEM_ERROR (out of
memory). Simply save the error message as a static string. If there is an
allocation failure constructing the error message, then convert the error to
out of memory. */
void ZLIB_INTERNAL gz_error(state, err, msg)
gz_statep state;
int err;
const char *msg;
{
/* free previously allocated message and clear */
if (state->msg != NULL) {
if (state->err != Z_MEM_ERROR)
free(state->msg);
state->msg = NULL;
}
/* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */
if (err != Z_OK && err != Z_BUF_ERROR)
state->x.have = 0;
/* set error code, and if no message, then done */
state->err = err;
if (msg == NULL)
return;
/* for an out of memory error, return literal string when requested */
if (err == Z_MEM_ERROR)
return;
/* construct error message with path */
if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) ==
NULL) {
state->err = Z_MEM_ERROR;
return;
}
#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
snprintf(state->msg, strlen(state->path) + strlen(msg) + 3,
"%s%s%s", state->path, ": ", msg);
#else
strcpy(state->msg, state->path);
strcat(state->msg, ": ");
strcat(state->msg, msg);
#endif
return;
}
#ifndef INT_MAX
/* portably return maximum value for an int (when limits.h presumed not
available) -- we need to do this to cover cases where 2's complement not
used, since C standard permits 1's complement and sign-bit representations,
otherwise we could just use ((unsigned)-1) >> 1 */
unsigned ZLIB_INTERNAL gz_intmax()
{
unsigned p, q;
p = 1;
do {
q = p;
p <<= 1;
p++;
} while (p > q);
return q >> 1;
}
#endif

594
fdbrpc/zlib/gzread.c
View File

@ -1,594 +0,0 @@
/* gzread.c -- zlib functions for reading gzip files
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "gzguts.h"
/* Local functions */
local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *));
local int gz_avail OF((gz_statep));
local int gz_look OF((gz_statep));
local int gz_decomp OF((gz_statep));
local int gz_fetch OF((gz_statep));
local int gz_skip OF((gz_statep, z_off64_t));
/* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from
state->fd, and update state->eof, state->err, and state->msg as appropriate.
This function needs to loop on read(), since read() is not guaranteed to
read the number of bytes requested, depending on the type of descriptor. */
local int gz_load(state, buf, len, have)
gz_statep state;
unsigned char *buf;
unsigned len;
unsigned *have;
{
int ret;
*have = 0;
do {
ret = read(state->fd, buf + *have, len - *have);
if (ret <= 0)
break;
*have += ret;
} while (*have < len);
if (ret < 0) {
gz_error(state, Z_ERRNO, zstrerror());
return -1;
}
if (ret == 0)
state->eof = 1;
return 0;
}
/* Load up input buffer and set eof flag if last data loaded -- return -1 on
error, 0 otherwise. Note that the eof flag is set when the end of the input
file is reached, even though there may be unused data in the buffer. Once
that data has been used, no more attempts will be made to read the file.
If strm->avail_in != 0, then the current data is moved to the beginning of
the input buffer, and then the remainder of the buffer is loaded with the
available data from the input file. */
local int gz_avail(state)
gz_statep state;
{
unsigned got;
z_streamp strm = &(state->strm);
if (state->err != Z_OK && state->err != Z_BUF_ERROR)
return -1;
if (state->eof == 0) {
if (strm->avail_in) { /* copy what's there to the start */
unsigned char *p = state->in;
unsigned const char *q = strm->next_in;
unsigned n = strm->avail_in;
do {
*p++ = *q++;
} while (--n);
}
if (gz_load(state, state->in + strm->avail_in,
state->size - strm->avail_in, &got) == -1)
return -1;
strm->avail_in += got;
strm->next_in = state->in;
}
return 0;
}
/* Look for gzip header, set up for inflate or copy. state->x.have must be 0.
If this is the first time in, allocate required memory. state->how will be
left unchanged if there is no more input data available, will be set to COPY
if there is no gzip header and direct copying will be performed, or it will
be set to GZIP for decompression. If direct copying, then leftover input
data from the input buffer will be copied to the output buffer. In that
case, all further file reads will be directly to either the output buffer or
a user buffer. If decompressing, the inflate state will be initialized.
gz_look() will return 0 on success or -1 on failure. */
local int gz_look(state)
gz_statep state;
{
z_streamp strm = &(state->strm);
/* allocate read buffers and inflate memory */
if (state->size == 0) {
/* allocate buffers */
state->in = (unsigned char *)malloc(state->want);
state->out = (unsigned char *)malloc(state->want << 1);
if (state->in == NULL || state->out == NULL) {
if (state->out != NULL)
free(state->out);
if (state->in != NULL)
free(state->in);
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
state->size = state->want;
/* allocate inflate memory */
state->strm.zalloc = Z_NULL;
state->strm.zfree = Z_NULL;
state->strm.opaque = Z_NULL;
state->strm.avail_in = 0;
state->strm.next_in = Z_NULL;
if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */
free(state->out);
free(state->in);
state->size = 0;
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
}
/* get at least the magic bytes in the input buffer */
if (strm->avail_in < 2) {
if (gz_avail(state) == -1)
return -1;
if (strm->avail_in == 0)
return 0;
}
/* look for gzip magic bytes -- if there, do gzip decoding (note: there is
a logical dilemma here when considering the case of a partially written
gzip file, to wit, if a single 31 byte is written, then we cannot tell
whether this is a single-byte file, or just a partially written gzip
file -- for here we assume that if a gzip file is being written, then
the header will be written in a single operation, so that reading a
single byte is sufficient indication that it is not a gzip file) */
if (strm->avail_in > 1 &&
strm->next_in[0] == 31 && strm->next_in[1] == 139) {
inflateReset(strm);
state->how = GZIP;
state->direct = 0;
return 0;
}
/* no gzip header -- if we were decoding gzip before, then this is trailing
garbage. Ignore the trailing garbage and finish. */
if (state->direct == 0) {
strm->avail_in = 0;
state->eof = 1;
state->x.have = 0;
return 0;
}
/* doing raw i/o, copy any leftover input to output -- this assumes that
the output buffer is larger than the input buffer, which also assures
space for gzungetc() */
state->x.next = state->out;
if (strm->avail_in) {
memcpy(state->x.next, strm->next_in, strm->avail_in);
state->x.have = strm->avail_in;
strm->avail_in = 0;
}
state->how = COPY;
state->direct = 1;
return 0;
}
/* Decompress from input to the provided next_out and avail_out in the state.
On return, state->x.have and state->x.next point to the just decompressed
data. If the gzip stream completes, state->how is reset to LOOK to look for
the next gzip stream or raw data, once state->x.have is depleted. Returns 0
on success, -1 on failure. */
local int gz_decomp(state)
gz_statep state;
{
int ret = Z_OK;
unsigned had;
z_streamp strm = &(state->strm);
/* fill output buffer up to end of deflate stream */
had = strm->avail_out;
do {
/* get more input for inflate() */
if (strm->avail_in == 0 && gz_avail(state) == -1)
return -1;
if (strm->avail_in == 0) {
gz_error(state, Z_BUF_ERROR, "unexpected end of file");
break;
}
/* decompress and handle errors */
ret = inflate(strm, Z_NO_FLUSH);
if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) {
gz_error(state, Z_STREAM_ERROR,
"internal error: inflate stream corrupt");
return -1;
}
if (ret == Z_MEM_ERROR) {
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
if (ret == Z_DATA_ERROR) { /* deflate stream invalid */
gz_error(state, Z_DATA_ERROR,
strm->msg == NULL ? "compressed data error" : strm->msg);
return -1;
}
} while (strm->avail_out && ret != Z_STREAM_END);
/* update available output */
state->x.have = had - strm->avail_out;
state->x.next = strm->next_out - state->x.have;
/* if the gzip stream completed successfully, look for another */
if (ret == Z_STREAM_END)
state->how = LOOK;
/* good decompression */
return 0;
}
/* Fetch data and put it in the output buffer. Assumes state->x.have is 0.
Data is either copied from the input file or decompressed from the input
file depending on state->how. If state->how is LOOK, then a gzip header is
looked for to determine whether to copy or decompress. Returns -1 on error,
otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the
end of the input file has been reached and all data has been processed. */
local int gz_fetch(state)
gz_statep state;
{
z_streamp strm = &(state->strm);
do {
switch(state->how) {
case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */
if (gz_look(state) == -1)
return -1;
if (state->how == LOOK)
return 0;
break;
case COPY: /* -> COPY */
if (gz_load(state, state->out, state->size << 1, &(state->x.have))
== -1)
return -1;
state->x.next = state->out;
return 0;
case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */
strm->avail_out = state->size << 1;
strm->next_out = state->out;
if (gz_decomp(state) == -1)
return -1;
}
} while (state->x.have == 0 && (!state->eof || strm->avail_in));
return 0;
}
/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */
local int gz_skip(state, len)
gz_statep state;
z_off64_t len;
{
unsigned n;
/* skip over len bytes or reach end-of-file, whichever comes first */
while (len)
/* skip over whatever is in output buffer */
if (state->x.have) {
n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ?
(unsigned)len : state->x.have;
state->x.have -= n;
state->x.next += n;
state->x.pos += n;
len -= n;
}
/* output buffer empty -- return if we're at the end of the input */
else if (state->eof && state->strm.avail_in == 0)
break;
/* need more data to skip -- load up output buffer */
else {
/* get more output, looking for header if required */
if (gz_fetch(state) == -1)
return -1;
}
return 0;
}
/* -- see zlib.h -- */
int ZEXPORT gzread(file, buf, len)
gzFile file;
voidp buf;
unsigned len;
{
unsigned got, n;
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're reading and that there's no (serious) error */
if (state->mode != GZ_READ ||
(state->err != Z_OK && state->err != Z_BUF_ERROR))
return -1;
/* since an int is returned, make sure len fits in one, otherwise return
with an error (this avoids the flaw in the interface) */
if ((int)len < 0) {
gz_error(state, Z_DATA_ERROR, "requested length does not fit in int");
return -1;
}
/* if len is zero, avoid unnecessary operations */
if (len == 0)
return 0;
/* process a skip request */
if (state->seek) {
state->seek = 0;
if (gz_skip(state, state->skip) == -1)
return -1;
}
/* get len bytes to buf, or less than len if at the end */
got = 0;
do {
/* first just try copying data from the output buffer */
if (state->x.have) {
n = state->x.have > len ? len : state->x.have;
memcpy(buf, state->x.next, n);
state->x.next += n;
state->x.have -= n;
}
/* output buffer empty -- return if we're at the end of the input */
else if (state->eof && strm->avail_in == 0) {
state->past = 1; /* tried to read past end */
break;
}
/* need output data -- for small len or new stream load up our output
buffer */
else if (state->how == LOOK || len < (state->size << 1)) {
/* get more output, looking for header if required */
if (gz_fetch(state) == -1)
return -1;
continue; /* no progress yet -- go back to copy above */
/* the copy above assures that we will leave with space in the
output buffer, allowing at least one gzungetc() to succeed */
}
/* large len -- read directly into user buffer */
else if (state->how == COPY) { /* read directly */
if (gz_load(state, (unsigned char *)buf, len, &n) == -1)
return -1;
}
/* large len -- decompress directly into user buffer */
else { /* state->how == GZIP */
strm->avail_out = len;
strm->next_out = (unsigned char *)buf;
if (gz_decomp(state) == -1)
return -1;
n = state->x.have;
state->x.have = 0;
}
/* update progress */
len -= n;
buf = (char *)buf + n;
got += n;
state->x.pos += n;
} while (len);
/* return number of bytes read into user buffer (will fit in int) */
return (int)got;
}
/* -- see zlib.h -- */
#ifdef Z_PREFIX_SET
# undef z_gzgetc
#else
# undef gzgetc
#endif
int ZEXPORT gzgetc(file)
gzFile file;
{
int ret;
unsigned char buf[1];
gz_statep state;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
/* check that we're reading and that there's no (serious) error */
if (state->mode != GZ_READ ||
(state->err != Z_OK && state->err != Z_BUF_ERROR))
return -1;
/* try output buffer (no need to check for skip request) */
if (state->x.have) {
state->x.have--;
state->x.pos++;
return *(state->x.next)++;
}
/* nothing there -- try gzread() */
ret = gzread(file, buf, 1);
return ret < 1 ? -1 : buf[0];
}
int ZEXPORT gzgetc_(file)
gzFile file;
{
return gzgetc(file);
}
/* -- see zlib.h -- */
int ZEXPORT gzungetc(c, file)
int c;
gzFile file;
{
gz_statep state;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
/* check that we're reading and that there's no (serious) error */
if (state->mode != GZ_READ ||
(state->err != Z_OK && state->err != Z_BUF_ERROR))
return -1;
/* process a skip request */
if (state->seek) {
state->seek = 0;
if (gz_skip(state, state->skip) == -1)
return -1;
}
/* can't push EOF */
if (c < 0)
return -1;
/* if output buffer empty, put byte at end (allows more pushing) */
if (state->x.have == 0) {
state->x.have = 1;
state->x.next = state->out + (state->size << 1) - 1;
state->x.next[0] = c;
state->x.pos--;
state->past = 0;
return c;
}
/* if no room, give up (must have already done a gzungetc()) */
if (state->x.have == (state->size << 1)) {
gz_error(state, Z_DATA_ERROR, "out of room to push characters");
return -1;
}
/* slide output data if needed and insert byte before existing data */
if (state->x.next == state->out) {
unsigned char *src = state->out + state->x.have;
unsigned char *dest = state->out + (state->size << 1);
while (src > state->out)
*--dest = *--src;
state->x.next = dest;
}
state->x.have++;
state->x.next--;
state->x.next[0] = c;
state->x.pos--;
state->past = 0;
return c;
}
/* -- see zlib.h -- */
char * ZEXPORT gzgets(file, buf, len)
gzFile file;
char *buf;
int len;
{
unsigned left, n;
char *str;
unsigned char *eol;
gz_statep state;
/* check parameters and get internal structure */
if (file == NULL || buf == NULL || len < 1)
return NULL;
state = (gz_statep)file;
/* check that we're reading and that there's no (serious) error */
if (state->mode != GZ_READ ||
(state->err != Z_OK && state->err != Z_BUF_ERROR))
return NULL;
/* process a skip request */
if (state->seek) {
state->seek = 0;
if (gz_skip(state, state->skip) == -1)
return NULL;
}
/* copy output bytes up to new line or len - 1, whichever comes first --
append a terminating zero to the string (we don't check for a zero in
the contents, let the user worry about that) */
str = buf;
left = (unsigned)len - 1;
if (left) do {
/* assure that something is in the output buffer */
if (state->x.have == 0 && gz_fetch(state) == -1)
return NULL; /* error */
if (state->x.have == 0) { /* end of file */
state->past = 1; /* read past end */
break; /* return what we have */
}
/* look for end-of-line in current output buffer */
n = state->x.have > left ? left : state->x.have;
eol = (unsigned char *)memchr(state->x.next, '\n', n);
if (eol != NULL)
n = (unsigned)(eol - state->x.next) + 1;
/* copy through end-of-line, or remainder if not found */
memcpy(buf, state->x.next, n);
state->x.have -= n;
state->x.next += n;
state->x.pos += n;
left -= n;
buf += n;
} while (left && eol == NULL);
/* return terminated string, or if nothing, end of file */
if (buf == str)
return NULL;
buf[0] = 0;
return str;
}
/* -- see zlib.h -- */
int ZEXPORT gzdirect(file)
gzFile file;
{
gz_statep state;
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
/* if the state is not known, but we can find out, then do so (this is
mainly for right after a gzopen() or gzdopen()) */
if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0)
(void)gz_look(state);
/* return 1 if transparent, 0 if processing a gzip stream */
return state->direct;
}
/* -- see zlib.h -- */
int ZEXPORT gzclose_r(file)
gzFile file;
{
int ret, err;
gz_statep state;
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
/* check that we're reading */
if (state->mode != GZ_READ)
return Z_STREAM_ERROR;
/* free memory and close file */
if (state->size) {
inflateEnd(&(state->strm));
free(state->out);
free(state->in);
}
err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK;
gz_error(state, Z_OK, NULL);
free(state->path);
ret = close(state->fd);
free(state);
return ret ? Z_ERRNO : err;
}

577
fdbrpc/zlib/gzwrite.c
View File

@ -1,577 +0,0 @@
/* gzwrite.c -- zlib functions for writing gzip files
* Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "gzguts.h"
/* Local functions */
local int gz_init OF((gz_statep));
local int gz_comp OF((gz_statep, int));
local int gz_zero OF((gz_statep, z_off64_t));
/* Initialize state for writing a gzip file. Mark initialization by setting
state->size to non-zero. Return -1 on failure or 0 on success. */
local int gz_init(state)
gz_statep state;
{
int ret;
z_streamp strm = &(state->strm);
/* allocate input buffer */
state->in = (unsigned char *)malloc(state->want);
if (state->in == NULL) {
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
/* only need output buffer and deflate state if compressing */
if (!state->direct) {
/* allocate output buffer */
state->out = (unsigned char *)malloc(state->want);
if (state->out == NULL) {
free(state->in);
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
/* allocate deflate memory, set up for gzip compression */
strm->zalloc = Z_NULL;
strm->zfree = Z_NULL;
strm->opaque = Z_NULL;
ret = deflateInit2(strm, state->level, Z_DEFLATED,
MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy);
if (ret != Z_OK) {
free(state->out);
free(state->in);
gz_error(state, Z_MEM_ERROR, "out of memory");
return -1;
}
}
/* mark state as initialized */
state->size = state->want;
/* initialize write buffer if compressing */
if (!state->direct) {
strm->avail_out = state->size;
strm->next_out = state->out;
state->x.next = strm->next_out;
}
return 0;
}
/* Compress whatever is at avail_in and next_in and write to the output file.
Return -1 if there is an error writing to the output file, otherwise 0.
flush is assumed to be a valid deflate() flush value. If flush is Z_FINISH,
then the deflate() state is reset to start a new gzip stream. If gz->direct
is true, then simply write to the output file without compressing, and
ignore flush. */
local int gz_comp(state, flush)
gz_statep state;
int flush;
{
int ret, got;
unsigned have;
z_streamp strm = &(state->strm);
/* allocate memory if this is the first time through */
if (state->size == 0 && gz_init(state) == -1)
return -1;
/* write directly if requested */
if (state->direct) {
got = write(state->fd, strm->next_in, strm->avail_in);
if (got < 0 || (unsigned)got != strm->avail_in) {
gz_error(state, Z_ERRNO, zstrerror());
return -1;
}
strm->avail_in = 0;
return 0;
}
/* run deflate() on provided input until it produces no more output */
ret = Z_OK;
do {
/* write out current buffer contents if full, or if flushing, but if
doing Z_FINISH then don't write until we get to Z_STREAM_END */
if (strm->avail_out == 0 || (flush != Z_NO_FLUSH &&
(flush != Z_FINISH || ret == Z_STREAM_END))) {
have = (unsigned)(strm->next_out - state->x.next);
if (have && ((got = write(state->fd, state->x.next, have)) < 0 ||
(unsigned)got != have)) {
gz_error(state, Z_ERRNO, zstrerror());
return -1;
}
if (strm->avail_out == 0) {
strm->avail_out = state->size;
strm->next_out = state->out;
}
state->x.next = strm->next_out;
}
/* compress */
have = strm->avail_out;
ret = deflate(strm, flush);
if (ret == Z_STREAM_ERROR) {
gz_error(state, Z_STREAM_ERROR,
"internal error: deflate stream corrupt");
return -1;
}
have -= strm->avail_out;
} while (have);
/* if that completed a deflate stream, allow another to start */
if (flush == Z_FINISH)
deflateReset(strm);
/* all done, no errors */
return 0;
}
/* Compress len zeros to output. Return -1 on error, 0 on success. */
local int gz_zero(state, len)
gz_statep state;
z_off64_t len;
{
int first;
unsigned n;
z_streamp strm = &(state->strm);
/* consume whatever's left in the input buffer */
if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
return -1;
/* compress len zeros (len guaranteed > 0) */
first = 1;
while (len) {
n = GT_OFF(state->size) || (z_off64_t)state->size > len ?
(unsigned)len : state->size;
if (first) {
memset(state->in, 0, n);
first = 0;
}
strm->avail_in = n;
strm->next_in = state->in;
state->x.pos += n;
if (gz_comp(state, Z_NO_FLUSH) == -1)
return -1;
len -= n;
}
return 0;
}
/* -- see zlib.h -- */
int ZEXPORT gzwrite(file, buf, len)
gzFile file;
voidpc buf;
unsigned len;
{
unsigned put = len;
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return 0;
/* since an int is returned, make sure len fits in one, otherwise return
with an error (this avoids the flaw in the interface) */
if ((int)len < 0) {
gz_error(state, Z_DATA_ERROR, "requested length does not fit in int");
return 0;
}
/* if len is zero, avoid unnecessary operations */
if (len == 0)
return 0;
/* allocate memory if this is the first time through */
if (state->size == 0 && gz_init(state) == -1)
return 0;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return 0;
}
/* for small len, copy to input buffer, otherwise compress directly */
if (len < state->size) {
/* copy to input buffer, compress when full */
do {
unsigned have, copy;
if (strm->avail_in == 0)
strm->next_in = state->in;
have = (unsigned)((strm->next_in + strm->avail_in) - state->in);
copy = state->size - have;
if (copy > len)
copy = len;
memcpy(state->in + have, buf, copy);
strm->avail_in += copy;
state->x.pos += copy;
buf = (const char *)buf + copy;
len -= copy;
if (len && gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
} while (len);
}
else {
/* consume whatever's left in the input buffer */
if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
/* directly compress user buffer to file */
strm->avail_in = len;
strm->next_in = (z_const Bytef *)buf;
state->x.pos += len;
if (gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
}
/* input was all buffered or compressed (put will fit in int) */
return (int)put;
}
/* -- see zlib.h -- */
int ZEXPORT gzputc(file, c)
gzFile file;
int c;
{
unsigned have;
unsigned char buf[1];
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return -1;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return -1;
}
/* try writing to input buffer for speed (state->size == 0 if buffer not
initialized) */
if (state->size) {
if (strm->avail_in == 0)
strm->next_in = state->in;
have = (unsigned)((strm->next_in + strm->avail_in) - state->in);
if (have < state->size) {
state->in[have] = c;
strm->avail_in++;
state->x.pos++;
return c & 0xff;
}
}
/* no room in buffer or not initialized, use gz_write() */
buf[0] = c;
if (gzwrite(file, buf, 1) != 1)
return -1;
return c & 0xff;
}
/* -- see zlib.h -- */
int ZEXPORT gzputs(file, str)
gzFile file;
const char *str;
{
int ret;
unsigned len;
/* write string */
len = (unsigned)strlen(str);
ret = gzwrite(file, str, len);
return ret == 0 && len != 0 ? -1 : ret;
}
#if defined(STDC) || defined(Z_HAVE_STDARG_H)
#include <stdarg.h>
/* -- see zlib.h -- */
int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
{
int size, len;
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return 0;
/* make sure we have some buffer space */
if (state->size == 0 && gz_init(state) == -1)
return 0;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return 0;
}
/* consume whatever's left in the input buffer */
if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
/* do the printf() into the input buffer, put length in len */
size = (int)(state->size);
state->in[size - 1] = 0;
#ifdef NO_vsnprintf
# ifdef HAS_vsprintf_void
(void)vsprintf((char *)(state->in), format, va);
for (len = 0; len < size; len++)
if (state->in[len] == 0) break;
# else
len = vsprintf((char *)(state->in), format, va);
# endif
#else
# ifdef HAS_vsnprintf_void
(void)vsnprintf((char *)(state->in), size, format, va);
len = strlen((char *)(state->in));
# else
len = vsnprintf((char *)(state->in), size, format, va);
# endif
#endif
/* check that printf() results fit in buffer */
if (len <= 0 || len >= (int)size || state->in[size - 1] != 0)
return 0;
/* update buffer and position, defer compression until needed */
strm->avail_in = (unsigned)len;
strm->next_in = state->in;
state->x.pos += len;
return len;
}
int ZEXPORTVA gzprintf(gzFile file, const char *format, ...)
{
va_list va;
int ret;
va_start(va, format);
ret = gzvprintf(file, format, va);
va_end(va);
return ret;
}
#else /* !STDC && !Z_HAVE_STDARG_H */
/* -- see zlib.h -- */
int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
a11, a12, a13, a14, a15, a16, a17, a18, a19, a20)
gzFile file;
const char *format;
int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
a11, a12, a13, a14, a15, a16, a17, a18, a19, a20;
{
int size, len;
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
strm = &(state->strm);
/* check that can really pass pointer in ints */
if (sizeof(int) != sizeof(void *))
return 0;
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return 0;
/* make sure we have some buffer space */
if (state->size == 0 && gz_init(state) == -1)
return 0;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return 0;
}
/* consume whatever's left in the input buffer */
if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
/* do the printf() into the input buffer, put length in len */
size = (int)(state->size);
state->in[size - 1] = 0;
#ifdef NO_snprintf
# ifdef HAS_sprintf_void
sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8,
a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
for (len = 0; len < size; len++)
if (state->in[len] == 0) break;
# else
len = sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8,
a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
# endif
#else
# ifdef HAS_snprintf_void
snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8,
a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20);
len = strlen((char *)(state->in));
# else
len = snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6,
a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18,
a19, a20);
# endif
#endif
/* check that printf() results fit in buffer */
if (len <= 0 || len >= (int)size || state->in[size - 1] != 0)
return 0;
/* update buffer and position, defer compression until needed */
strm->avail_in = (unsigned)len;
strm->next_in = state->in;
state->x.pos += len;
return len;
}
#endif
/* -- see zlib.h -- */
int ZEXPORT gzflush(file, flush)
gzFile file;
int flush;
{
gz_statep state;
/* get internal structure */
if (file == NULL)
return -1;
state = (gz_statep)file;
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return Z_STREAM_ERROR;
/* check flush parameter */
if (flush < 0 || flush > Z_FINISH)
return Z_STREAM_ERROR;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return -1;
}
/* compress remaining data with requested flush */
gz_comp(state, flush);
return state->err;
}
/* -- see zlib.h -- */
int ZEXPORT gzsetparams(file, level, strategy)
gzFile file;
int level;
int strategy;
{
gz_statep state;
z_streamp strm;
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
strm = &(state->strm);
/* check that we're writing and that there's no error */
if (state->mode != GZ_WRITE || state->err != Z_OK)
return Z_STREAM_ERROR;
/* if no change is requested, then do nothing */
if (level == state->level && strategy == state->strategy)
return Z_OK;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
return -1;
}
/* change compression parameters for subsequent input */
if (state->size) {
/* flush previous input with previous parameters before changing */
if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1)
return state->err;
deflateParams(strm, level, strategy);
}
state->level = level;
state->strategy = strategy;
return Z_OK;
}
/* -- see zlib.h -- */
int ZEXPORT gzclose_w(file)
gzFile file;
{
int ret = Z_OK;
gz_statep state;
/* get internal structure */
if (file == NULL)
return Z_STREAM_ERROR;
state = (gz_statep)file;
/* check that we're writing */
if (state->mode != GZ_WRITE)
return Z_STREAM_ERROR;
/* check for seek request */
if (state->seek) {
state->seek = 0;
if (gz_zero(state, state->skip) == -1)
ret = state->err;
}
/* flush, free memory, and close file */
if (gz_comp(state, Z_FINISH) == -1)
ret = state->err;
if (state->size) {
if (!state->direct) {
(void)deflateEnd(&(state->strm));
free(state->out);
}
free(state->in);
}
gz_error(state, Z_OK, NULL);
free(state->path);
if (close(state->fd) == -1)
ret = Z_ERRNO;
free(state);
return ret;
}

640
fdbrpc/zlib/infback.c
View File

@ -1,640 +0,0 @@
/* infback.c -- inflate using a call-back interface
* Copyright (C) 1995-2011 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/*
This code is largely copied from inflate.c. Normally either infback.o or
inflate.o would be linked into an application--not both. The interface
with inffast.c is retained so that optimized assembler-coded versions of
inflate_fast() can be used with either inflate.c or infback.c.
*/
#include "zutil.h"
#include "inftrees.h"
#include "inflate.h"
#include "inffast.h"
/* function prototypes */
local void fixedtables OF((struct inflate_state FAR *state));
/*
strm provides memory allocation functions in zalloc and zfree, or
Z_NULL to use the library memory allocation functions.
windowBits is in the range 8..15, and window is a user-supplied
window and output buffer that is 2**windowBits bytes.
*/
int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size)
z_streamp strm;
int windowBits;
unsigned char FAR *window;
const char *version;
int stream_size;
{
struct inflate_state FAR *state;
if (version == Z_NULL || version[0] != ZLIB_VERSION[0] ||
stream_size != (int)(sizeof(z_stream)))
return Z_VERSION_ERROR;
if (strm == Z_NULL || window == Z_NULL ||
windowBits < 8 || windowBits > 15)
return Z_STREAM_ERROR;
strm->msg = Z_NULL; /* in case we return an error */
if (strm->zalloc == (alloc_func)0) {
#ifdef Z_SOLO
return Z_STREAM_ERROR;
#else
strm->zalloc = zcalloc;
strm->opaque = (voidpf)0;
#endif
}
if (strm->zfree == (free_func)0)
#ifdef Z_SOLO
return Z_STREAM_ERROR;
#else
strm->zfree = zcfree;
#endif
state = (struct inflate_state FAR *)ZALLOC(strm, 1,
sizeof(struct inflate_state));
if (state == Z_NULL) return Z_MEM_ERROR;
Tracev((stderr, "inflate: allocated\n"));
strm->state = (struct internal_state FAR *)state;
state->dmax = 32768U;
state->wbits = windowBits;
state->wsize = 1U << windowBits;
state->window = window;
state->wnext = 0;
state->whave = 0;
return Z_OK;
}
/*
Return state with length and distance decoding tables and index sizes set to
fixed code decoding. Normally this returns fixed tables from inffixed.h.
If BUILDFIXED is defined, then instead this routine builds the tables the
first time it's called, and returns those tables the first time and
thereafter. This reduces the size of the code by about 2K bytes, in
exchange for a little execution time. However, BUILDFIXED should not be
used for threaded applications, since the rewriting of the tables and virgin
may not be thread-safe.
*/
local void fixedtables(state)
struct inflate_state FAR *state;
{
#ifdef BUILDFIXED
static int virgin = 1;
static code *lenfix, *distfix;
static code fixed[544];
/* build fixed huffman tables if first call (may not be thread safe) */
if (virgin) {
unsigned sym, bits;
static code *next;
/* literal/length table */
sym = 0;
while (sym < 144) state->lens[sym++] = 8;
while (sym < 256) state->lens[sym++] = 9;
while (sym < 280) state->lens[sym++] = 7;
while (sym < 288) state->lens[sym++] = 8;
next = fixed;
lenfix = next;
bits = 9;
inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work);
/* distance table */
sym = 0;
while (sym < 32) state->lens[sym++] = 5;
distfix = next;
bits = 5;
inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work);
/* do this just once */
virgin = 0;
}
#else /* !BUILDFIXED */
# include "inffixed.h"
#endif /* BUILDFIXED */
state->lencode = lenfix;
state->lenbits = 9;
state->distcode = distfix;
state->distbits = 5;
}
/* Macros for inflateBack(): */
/* Load returned state from inflate_fast() */
#define LOAD() \
do { \
put = strm->next_out; \
left = strm->avail_out; \
next = strm->next_in; \
have = strm->avail_in; \
hold = state->hold; \
bits = state->bits; \
} while (0)
/* Set state from registers for inflate_fast() */
#define RESTORE() \
do { \
strm->next_out = put; \
strm->avail_out = left; \
strm->next_in = next; \
strm->avail_in = have; \
state->hold = hold; \
state->bits = bits; \
} while (0)
/* Clear the input bit accumulator */
#define INITBITS() \
do { \
hold = 0; \
bits = 0; \
} while (0)
/* Assure that some input is available. If input is requested, but denied,
then return a Z_BUF_ERROR from inflateBack(). */
#define PULL() \
do { \
if (have == 0) { \
have = in(in_desc, &next); \
if (have == 0) { \
next = Z_NULL; \
ret = Z_BUF_ERROR; \
goto inf_leave; \
} \
} \
} while (0)
/* Get a byte of input into the bit accumulator, or return from inflateBack()
with an error if there is no input available. */
#define PULLBYTE() \
do { \
PULL(); \
have--; \
hold += (unsigned long)(*next++) << bits; \
bits += 8; \
} while (0)
/* Assure that there are at least n bits in the bit accumulator. If there is
not enough available input to do that, then return from inflateBack() with
an error. */
#define NEEDBITS(n) \
do { \
while (bits < (unsigned)(n)) \
PULLBYTE(); \
} while (0)
/* Return the low n bits of the bit accumulator (n < 16) */
#define BITS(n) \
((unsigned)hold & ((1U << (n)) - 1))
/* Remove n bits from the bit accumulator */
#define DROPBITS(n) \
do { \
hold >>= (n); \
bits -= (unsigned)(n); \
} while (0)
/* Remove zero to seven bits as needed to go to a byte boundary */
#define BYTEBITS() \
do { \
hold >>= bits & 7; \
bits -= bits & 7; \
} while (0)
/* Assure that some output space is available, by writing out the window
if it's full. If the write fails, return from inflateBack() with a
Z_BUF_ERROR. */
#define ROOM() \
do { \
if (left == 0) { \
put = state->window; \
left = state->wsize; \
state->whave = left; \
if (out(out_desc, put, left)) { \
ret = Z_BUF_ERROR; \
goto inf_leave; \
} \
} \
} while (0)
/*
strm provides the memory allocation functions and window buffer on input,
and provides information on the unused input on return. For Z_DATA_ERROR
returns, strm will also provide an error message.
in() and out() are the call-back input and output functions. When
inflateBack() needs more input, it calls in(). When inflateBack() has
filled the window with output, or when it completes with data in the
window, it calls out() to write out the data. The application must not
change the provided input until in() is called again or inflateBack()
returns. The application must not change the window/output buffer until
inflateBack() returns.
in() and out() are called with a descriptor parameter provided in the
inflateBack() call. This parameter can be a structure that provides the
information required to do the read or write, as well as accumulated
information on the input and output such as totals and check values.
in() should return zero on failure. out() should return non-zero on
failure. If either in() or out() fails, than inflateBack() returns a
Z_BUF_ERROR. strm->next_in can be checked for Z_NULL to see whether it
was in() or out() that caused in the error. Otherwise, inflateBack()
returns Z_STREAM_END on success, Z_DATA_ERROR for an deflate format
error, or Z_MEM_ERROR if it could not allocate memory for the state.
inflateBack() can also return Z_STREAM_ERROR if the input parameters
are not correct, i.e. strm is Z_NULL or the state was not initialized.
*/
int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc)
z_streamp strm;
in_func in;
void FAR *in_desc;
out_func out;
void FAR *out_desc;
{
struct inflate_state FAR *state;
z_const unsigned char FAR *next; /* next input */
unsigned char FAR *put; /* next output */
unsigned have, left; /* available input and output */
unsigned long hold; /* bit buffer */
unsigned bits; /* bits in bit buffer */
unsigned copy; /* number of stored or match bytes to copy */
unsigned char FAR *from; /* where to copy match bytes from */
code here; /* current decoding table entry */
code last; /* parent table entry */
unsigned len; /* length to copy for repeats, bits to drop */
int ret; /* return code */
static const unsigned short order[19] = /* permutation of code lengths */
{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
/* Check that the strm exists and that the state was initialized */
if (strm == Z_NULL || strm->state == Z_NULL)
return Z_STREAM_ERROR;
state = (struct inflate_state FAR *)strm->state;
/* Reset the state */
strm->msg = Z_NULL;
state->mode = TYPE;
state->last = 0;
state->whave = 0;
next = strm->next_in;
have = next != Z_NULL ? strm->avail_in : 0;
hold = 0;
bits = 0;
put = state->window;
left = state->wsize;
/* Inflate until end of block marked as last */
for (;;)
switch (state->mode) {
case TYPE:
/* determine and dispatch block type */
if (state->last) {
BYTEBITS();
state->mode = DONE;
break;
}
NEEDBITS(3);
state->last = BITS(1);
DROPBITS(1);
switch (BITS(2)) {
case 0: /* stored block */
Tracev((stderr, "inflate: stored block%s\n",
state->last ? " (last)" : ""));
state->mode = STORED;
break;
case 1: /* fixed block */
fixedtables(state);
Tracev((stderr, "inflate: fixed codes block%s\n",
state->last ? " (last)" : ""));
state->mode = LEN; /* decode codes */
break;
case 2: /* dynamic block */
Tracev((stderr, "inflate: dynamic codes block%s\n",
state->last ? " (last)" : ""));
state->mode = TABLE;
break;
case 3:
strm->msg = (char *)"invalid block type";
state->mode = BAD;
}
DROPBITS(2);
break;
case STORED:
/* get and verify stored block length */
BYTEBITS(); /* go to byte boundary */
NEEDBITS(32);
if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) {
strm->msg = (char *)"invalid stored block lengths";
state->mode = BAD;
break;
}
state->length = (unsigned)hold & 0xffff;
Tracev((stderr, "inflate: stored length %u\n",
state->length));
INITBITS();
/* copy stored block from input to output */
while (state->length != 0) {
copy = state->length;
PULL();
ROOM();
if (copy > have) copy = have;
if (copy > left) copy = left;
zmemcpy(put, next, copy);
have -= copy;
next += copy;
left -= copy;
put += copy;
state->length -= copy;
}
Tracev((stderr, "inflate: stored end\n"));
state->mode = TYPE;
break;
case TABLE:
/* get dynamic table entries descriptor */
NEEDBITS(14);
state->nlen = BITS(5) + 257;
DROPBITS(5);
state->ndist = BITS(5) + 1;
DROPBITS(5);
state->ncode = BITS(4) + 4;
DROPBITS(4);
#ifndef PKZIP_BUG_WORKAROUND
if (state->nlen > 286 || state->ndist > 30) {
strm->msg = (char *)"too many length or distance symbols";
state->mode = BAD;
break;
}
#endif
Tracev((stderr, "inflate: table sizes ok\n"));
/* get code length code lengths (not a typo) */
state->have = 0;
while (state->have < state->ncode) {
NEEDBITS(3);
state->lens[order[state->have++]] = (unsigned short)BITS(3);
DROPBITS(3);
}
while (state->have < 19)
state->lens[order[state->have++]] = 0;
state->next = state->codes;
state->lencode = (code const FAR *)(state->next);
state->lenbits = 7;
ret = inflate_table(CODES, state->lens, 19, &(state->next),
&(state->lenbits), state->work);
if (ret) {
strm->msg = (char *)"invalid code lengths set";
state->mode = BAD;
break;
}
Tracev((stderr, "inflate: code lengths ok\n"));
/* get length and distance code code lengths */
state->have = 0;
while (state->have < state->nlen + state->ndist) {
for (;;) {
here = state->lencode[BITS(state->lenbits)];
if ((unsigned)(here.bits) <= bits) break;
PULLBYTE();
}
if (here.val < 16) {
DROPBITS(here.bits);
state->lens[state->have++] = here.val;
}
else {
if (here.val == 16) {
NEEDBITS(here.bits + 2);
DROPBITS(here.bits);
if (state->have == 0) {
strm->msg = (char *)"invalid bit length repeat";
state->mode = BAD;
break;
}
len = (unsigned)(state->lens[state->have - 1]);
copy = 3 + BITS(2);
DROPBITS(2);
}
else if (here.val == 17) {
NEEDBITS(here.bits + 3);
DROPBITS(here.bits);
len = 0;
copy = 3 + BITS(3);
DROPBITS(3);
}
else {
NEEDBITS(here.bits + 7);
DROPBITS(here.bits);
len = 0;
copy = 11 + BITS(7);
DROPBITS(7);
}
if (state->have + copy > state->nlen + state->ndist) {
strm->msg = (char *)"invalid bit length repeat";
state->mode = BAD;
break;
}
while (copy--)
state->lens[state->have++] = (unsigned short)len;
}
}
/* handle error breaks in while */
if (state->mode == BAD) break;
/* check for end-of-block code (better have one) */
if (state->lens[256] == 0) {
strm->msg = (char *)"invalid code -- missing end-of-block";
state->mode = BAD;
break;
}
/* build code tables -- note: do not change the lenbits or distbits
values here (9 and 6) without reading the comments in inftrees.h
concerning the ENOUGH constants, which depend on those values */
state->next = state->codes;
state->lencode = (code const FAR *)(state->next);
state->lenbits = 9;
ret = inflate_table(LENS, state->lens, state->nlen, &(state->next),
&(state->lenbits), state->work);
if (ret) {
strm->msg = (char *)"invalid literal/lengths set";
state->mode = BAD;
break;
}
state->distcode = (code const FAR *)(state->next);
state->distbits = 6;
ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist,
&(state->next), &(state->distbits), state->work);
if (ret) {
strm->msg = (char *)"invalid distances set";
state->mode = BAD;
break;
}
Tracev((stderr, "inflate: codes ok\n"));
state->mode = LEN;
case LEN:
/* use inflate_fast() if we have enough input and output */
if (have >= 6 && left >= 258) {
RESTORE();
if (state->whave < state->wsize)
state->whave = state->wsize - left;
inflate_fast(strm, state->wsize);
LOAD();
break;
}
/* get a literal, length, or end-of-block code */
for (;;) {
here = state->lencode[BITS(state->lenbits)];
if ((unsigned)(here.bits) <= bits) break;
PULLBYTE();
}
if (here.op && (here.op & 0xf0) == 0) {
last = here;
for (;;) {
here = state->lencode[last.val +
(BITS(last.bits + last.op) >> last.bits)];
if ((unsigned)(last.bits + here.bits) <= bits) break;
PULLBYTE();
}
DROPBITS(last.bits);
}
DROPBITS(here.bits);
state->length = (unsigned)here.val;
/* process literal */
if (here.op == 0) {
Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ?
"inflate: literal '%c'\n" :
"inflate: literal 0x%02x\n", here.val));
ROOM();
*put++ = (unsigned char)(state->length);
left--;
state->mode = LEN;
break;
}
/* process end of block */
if (here.op & 32) {
Tracevv((stderr, "inflate: end of block\n"));
state->mode = TYPE;
break;
}
/* invalid code */
if (here.op & 64) {
strm->msg = (char *)"invalid literal/length code";
state->mode = BAD;
break;
}
/* length code -- get extra bits, if any */
state->extra = (unsigned)(here.op) & 15;
if (state->extra != 0) {
NEEDBITS(state->extra);
state->length += BITS(state->extra);
DROPBITS(state->extra);
}
Tracevv((stderr, "inflate: length %u\n", state->length));
/* get distance code */
for (;;) {
here = state->distcode[BITS(state->distbits)];
if ((unsigned)(here.bits) <= bits) break;
PULLBYTE();
}
if ((here.op & 0xf0) == 0) {
last = here;
for (;;) {
here = state->distcode[last.val +
(BITS(last.bits + last.op) >> last.bits)];
if ((unsigned)(last.bits + here.bits) <= bits) break;
PULLBYTE();
}
DROPBITS(last.bits);
}
DROPBITS(here.bits);
if (here.op & 64) {
strm->msg = (char *)"invalid distance code";
state->mode = BAD;
break;
}
state->offset = (unsigned)here.val;
/* get distance extra bits, if any */
state->extra = (unsigned)(here.op) & 15;
if (state->extra != 0) {
NEEDBITS(state->extra);
state->offset += BITS(state->extra);
DROPBITS(state->extra);
}
if (state->offset > state->wsize - (state->whave < state->wsize ?
left : 0)) {
strm->msg = (char *)"invalid distance too far back";
state->mode = BAD;
break;
}
Tracevv((stderr, "inflate: distance %u\n", state->offset));
/* copy match from window to output */
do {
ROOM();
copy = state->wsize - state->offset;
if (copy < left) {
from = put + copy;
copy = left - copy;
}
else {
from = put - state->offset;
copy = left;
}
if (copy > state->length) copy = state->length;
state->length -= copy;
left -= copy;
do {
*put++ = *from++;
} while (--copy);
} while (state->length != 0);
break;
case DONE:
/* inflate stream terminated properly -- write leftover output */
ret = Z_STREAM_END;
if (left < state->wsize) {
if (out(out_desc, state->window, state->wsize - left))
ret = Z_BUF_ERROR;
}
goto inf_leave;
case BAD:
ret = Z_DATA_ERROR;
goto inf_leave;
default: /* can't happen, but makes compilers happy */
ret = Z_STREAM_ERROR;
goto inf_leave;
}
/* Return unused input */
inf_leave:
strm->next_in = next;
strm->avail_in = have;
return ret;
}
int ZEXPORT inflateBackEnd(strm)
z_streamp strm;
{
if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0)
return Z_STREAM_ERROR;
ZFREE(strm, strm->state);
strm->state = Z_NULL;
Tracev((stderr, "inflate: end\n"));
return Z_OK;
}

340
fdbrpc/zlib/inffast.c
View File

@ -1,340 +0,0 @@
/* inffast.c -- fast decoding
* Copyright (C) 1995-2008, 2010, 2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "zutil.h"
#include "inftrees.h"
#include "inflate.h"
#include "inffast.h"
#ifndef ASMINF
/* Allow machine dependent optimization for post-increment or pre-increment.
Based on testing to date,
Pre-increment preferred for:
- PowerPC G3 (Adler)
- MIPS R5000 (Randers-Pehrson)
Post-increment preferred for:
- none
No measurable difference:
- Pentium III (Anderson)
- M68060 (Nikl)
*/
#ifdef POSTINC
# define OFF 0
# define PUP(a) *(a)++
#else
# define OFF 1
# define PUP(a) *++(a)
#endif
/*
Decode literal, length, and distance codes and write out the resulting
literal and match bytes until either not enough input or output is
available, an end-of-block is encountered, or a data error is encountered.
When large enough input and output buffers are supplied to inflate(), for
example, a 16K input buffer and a 64K output buffer, more than 95% of the
inflate execution time is spent in this routine.
Entry assumptions:
state->mode == LEN
strm->avail_in >= 6
strm->avail_out >= 258
start >= strm->avail_out
state->bits < 8
On return, state->mode is one of:
LEN -- ran out of enough output space or enough available input
TYPE -- reached end of block code, inflate() to interpret next block
BAD -- error in block data
Notes:
- The maximum input bits used by a length/distance pair is 15 bits for the
length code, 5 bits for the length extra, 15 bits for the distance code,
and 13 bits for the distance extra. This totals 48 bits, or six bytes.
Therefore if strm->avail_in >= 6, then there is enough input to avoid
checking for available input while decoding.
- The maximum bytes that a single length/distance pair can output is 258
bytes, which is the maximum length that can be coded. inflate_fast()
requires strm->avail_out >= 258 for each loop to avoid checking for
output space.
*/
void ZLIB_INTERNAL inflate_fast(strm, start)
z_streamp strm;
unsigned start; /* inflate()'s starting value for strm->avail_out */
{
struct inflate_state FAR *state;
z_const unsigned char FAR *in; /* local strm->next_in */
z_const unsigned char FAR *last; /* have enough input while in < last */
unsigned char FAR *out; /* local strm->next_out */
unsigned char FAR *beg; /* inflate()'s initial strm->next_out */
unsigned char FAR *end; /* while out < end, enough space available */
#ifdef INFLATE_STRICT
unsigned dmax; /* maximum distance from zlib header */
#endif
unsigned wsize; /* window size or zero if not using window */
unsigned whave; /* valid bytes in the window */
unsigned wnext; /* window write index */
unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */
unsigned long hold; /* local strm->hold */
unsigned bits; /* local strm->bits */
code const FAR *lcode; /* local strm->lencode */
code const FAR *dcode; /* local strm->distcode */
unsigned lmask; /* mask for first level of length codes */
unsigned dmask; /* mask for first level of distance codes */
code here; /* retrieved table entry */
unsigned op; /* code bits, operation, extra bits, or */
/* window position, window bytes to copy */
unsigned len; /* match length, unused bytes */
unsigned dist; /* match distance */
unsigned char FAR *from; /* where to copy match from */
/* copy state to local variables */
state = (struct inflate_state FAR *)strm->state;
in = strm->next_in - OFF;
last = in + (strm->avail_in - 5);
out = strm->next_out - OFF;
beg = out - (start - strm->avail_out);
end = out + (strm->avail_out - 257);
#ifdef INFLATE_STRICT
dmax = state->dmax;
#endif
wsize = state->wsize;
whave = state->whave;
wnext = state->wnext;
window = state->window;
hold = state->hold;
bits = state->bits;
lcode = state->lencode;
dcode = state->distcode;
lmask = (1U << state->lenbits) - 1;
dmask = (1U << state->distbits) - 1;
/* decode literals and length/distances until end-of-block or not enough
input data or output space */
do {
if (bits < 15) {
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
}
here = lcode[hold & lmask];
dolen:
op = (unsigned)(here.bits);
hold >>= op;
bits -= op;
op = (unsigned)(here.op);
if (op == 0) { /* literal */
Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ?
"inflate: literal '%c'\n" :
"inflate: literal 0x%02x\n", here.val));
PUP(out) = (unsigned char)(here.val);
}
else if (op & 16) { /* length base */
len = (unsigned)(here.val);
op &= 15; /* number of extra bits */
if (op) {
if (bits < op) {
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
}
len += (unsigned)hold & ((1U << op) - 1);
hold >>= op;
bits -= op;
}
Tracevv((stderr, "inflate: length %u\n", len));
if (bits < 15) {
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
}
here = dcode[hold & dmask];
dodist:
op = (unsigned)(here.bits);
hold >>= op;
bits -= op;
op = (unsigned)(here.op);
if (op & 16) { /* distance base */
dist = (unsigned)(here.val);
op &= 15; /* number of extra bits */
if (bits < op) {
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
if (bits < op) {
hold += (unsigned long)(PUP(in)) << bits;
bits += 8;
}
}
dist += (unsigned)hold & ((1U << op) - 1);
#ifdef INFLATE_STRICT
if (dist > dmax) {
strm->msg = (char *)"invalid distance too far back";
state->mode = BAD;
break;
}
#endif
hold >>= op;
bits -= op;
Tracevv((stderr, "inflate: distance %u\n", dist));
op = (unsigned)(out - beg); /* max distance in output */
if (dist > op) { /* see if copy from window */
op = dist - op; /* distance back in window */
if (op > whave) {
if (state->sane) {
strm->msg =
(char *)"invalid distance too far back";
state->mode = BAD;
break;
}
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
if (len <= op - whave) {
do {
PUP(out) = 0;
} while (--len);
continue;
}
len -= op - whave;
do {
PUP(out) = 0;
} while (--op > whave);
if (op == 0) {
from = out - dist;
do {
PUP(out) = PUP(from);
} while (--len);
continue;
}
#endif
}
from = window - OFF;
if (wnext == 0) { /* very common case */
from += wsize - op;
if (op < len) { /* some from window */
len -= op;
do {
PUP(out) = PUP(from);
} while (--op);
from = out - dist; /* rest from output */
}
}
else if (wnext < op) { /* wrap around window */
from += wsize + wnext - op;
op -= wnext;
if (op < len) { /* some from end of window */
len -= op;
do {
PUP(out) = PUP(from);
} while (--op);
from = window - OFF;
if (wnext < len) { /* some from start of window */
op = wnext;
len -= op;
do {
PUP(out) = PUP(from);
} while (--op);
from = out - dist; /* rest from output */
}
}
}
else { /* contiguous in window */
from += wnext - op;
if (op < len) { /* some from window */
len -= op;
do {
PUP(out) = PUP(from);
} while (--op);
from = out - dist; /* rest from output */
}
}
while (len > 2) {
PUP(out) = PUP(from);
PUP(out) = PUP(from);
PUP(out) = PUP(from);
len -= 3;
}
if (len) {
PUP(out) = PUP(from);
if (len > 1)
PUP(out) = PUP(from);
}
}
else {
from = out - dist; /* copy direct from output */
do { /* minimum length is three */
PUP(out) = PUP(from);
PUP(out) = PUP(from);
PUP(out) = PUP(from);
len -= 3;
} while (len > 2);
if (len) {
PUP(out) = PUP(from);
if (len > 1)
PUP(out) = PUP(from);
}
}
}
else if ((op & 64) == 0) { /* 2nd level distance code */
here = dcode[here.val + (hold & ((1U << op) - 1))];
goto dodist;
}
else {
strm->msg = (char *)"invalid distance code";
state->mode = BAD;
break;
}
}
else if ((op & 64) == 0) { /* 2nd level length code */
here = lcode[here.val + (hold & ((1U << op) - 1))];
goto dolen;
}
else if (op & 32) { /* end-of-block */
Tracevv((stderr, "inflate: end of block\n"));
state->mode = TYPE;
break;
}
else {
strm->msg = (char *)"invalid literal/length code";
state->mode = BAD;
break;
}
} while (in < last && out < end);
/* return unused bytes (on entry, bits < 8, so in won't go too far back) */
len = bits >> 3;
in -= len;
bits -= len << 3;
hold &= (1U << bits) - 1;
/* update state and return */
strm->next_in = in + OFF;
strm->next_out = out + OFF;
strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last));
strm->avail_out = (unsigned)(out < end ?
257 + (end - out) : 257 - (out - end));
state->hold = hold;
state->bits = bits;
return;
}
/*
inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
- Using bit fields for code structure
- Different op definition to avoid & for extra bits (do & for table bits)
- Three separate decoding do-loops for direct, window, and wnext == 0
- Special case for distance > 1 copies to do overlapped load and store copy
- Explicit branch predictions (based on measured branch probabilities)
- Deferring match copy and interspersed it with decoding subsequent codes
- Swapping literal/length else
- Swapping window/direct else
- Larger unrolled copy loops (three is about right)
- Moving len -= 3 statement into middle of loop
*/
#endif /* !ASMINF */

11
fdbrpc/zlib/inffast.h
View File

@ -1,11 +0,0 @@
/* inffast.h -- header to use inffast.c
* Copyright (C) 1995-2003, 2010 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* WARNING: this file should *not* be used by applications. It is
part of the implementation of the compression library and is
subject to change. Applications should only use zlib.h.
*/
void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start));

94
fdbrpc/zlib/inffixed.h
View File

@ -1,94 +0,0 @@
/* inffixed.h -- table for decoding fixed codes
* Generated automatically by makefixed().
*/
/* WARNING: this file should *not* be used by applications.
It is part of the implementation of this library and is
subject to change. Applications should only use zlib.h.
*/
static const code lenfix[512] = {
{96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48},
{0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128},
{0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59},
{0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176},
{0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20},
{21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100},
{0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8},
{0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216},
{18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76},
{0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114},
{0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2},
{0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148},
{20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42},
{0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86},
{0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15},
{0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236},
{16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62},
{0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142},
{0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31},
{0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162},
{0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25},
{0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105},
{0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4},
{0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202},
{17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69},
{0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125},
{0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13},
{0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195},
{19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35},
{0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91},
{0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19},
{0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246},
{16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55},
{0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135},
{0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99},
{0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190},
{0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16},
{20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96},
{0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6},
{0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209},
{17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72},
{0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116},
{0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4},
{0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153},
{20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44},
{0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82},
{0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11},
{0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229},
{16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58},
{0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138},
{0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51},
{0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173},
{0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30},
{0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110},
{0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0},
{0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195},
{16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65},
{0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121},
{0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9},
{0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258},
{19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37},
{0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93},
{0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23},
{0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251},
{16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51},
{0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131},
{0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67},
{0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183},
{0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23},
{64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103},
{0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9},
{0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223},
{18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79},
{0,9,255}
};
static const code distfix[32] = {
{16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025},
{21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193},
{18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385},
{19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577},
{16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073},
{22,5,193},{64,5,0}
};

1512
fdbrpc/zlib/inflate.c
View File

File diff suppressed because it is too large Load Diff

122
fdbrpc/zlib/inflate.h
View File

@ -1,122 +0,0 @@
/* inflate.h -- internal inflate state definition
* Copyright (C) 1995-2009 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* WARNING: this file should *not* be used by applications. It is
part of the implementation of the compression library and is
subject to change. Applications should only use zlib.h.
*/
/* define NO_GZIP when compiling if you want to disable gzip header and
trailer decoding by inflate(). NO_GZIP would be used to avoid linking in
the crc code when it is not needed. For shared libraries, gzip decoding
should be left enabled. */
#ifndef NO_GZIP
# define GUNZIP
#endif
/* Possible inflate modes between inflate() calls */
typedef enum {
HEAD, /* i: waiting for magic header */
FLAGS, /* i: waiting for method and flags (gzip) */
TIME, /* i: waiting for modification time (gzip) */
OS, /* i: waiting for extra flags and operating system (gzip) */
EXLEN, /* i: waiting for extra length (gzip) */
EXTRA, /* i: waiting for extra bytes (gzip) */
NAME, /* i: waiting for end of file name (gzip) */
COMMENT, /* i: waiting for end of comment (gzip) */
HCRC, /* i: waiting for header crc (gzip) */
DICTID, /* i: waiting for dictionary check value */
DICT, /* waiting for inflateSetDictionary() call */
TYPE, /* i: waiting for type bits, including last-flag bit */
TYPEDO, /* i: same, but skip check to exit inflate on new block */
STORED, /* i: waiting for stored size (length and complement) */
COPY_, /* i/o: same as COPY below, but only first time in */
COPY, /* i/o: waiting for input or output to copy stored block */
TABLE, /* i: waiting for dynamic block table lengths */
LENLENS, /* i: waiting for code length code lengths */
CODELENS, /* i: waiting for length/lit and distance code lengths */
LEN_, /* i: same as LEN below, but only first time in */
LEN, /* i: waiting for length/lit/eob code */
LENEXT, /* i: waiting for length extra bits */
DIST, /* i: waiting for distance code */
DISTEXT, /* i: waiting for distance extra bits */
MATCH, /* o: waiting for output space to copy string */
LIT, /* o: waiting for output space to write literal */
CHECK, /* i: waiting for 32-bit check value */
LENGTH, /* i: waiting for 32-bit length (gzip) */
DONE, /* finished check, done -- remain here until reset */
BAD, /* got a data error -- remain here until reset */
MEM, /* got an inflate() memory error -- remain here until reset */
SYNC /* looking for synchronization bytes to restart inflate() */
} inflate_mode;
/*
State transitions between above modes -
(most modes can go to BAD or MEM on error -- not shown for clarity)
Process header:
HEAD -> (gzip) or (zlib) or (raw)
(gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT ->
HCRC -> TYPE
(zlib) -> DICTID or TYPE
DICTID -> DICT -> TYPE
(raw) -> TYPEDO
Read deflate blocks:
TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK
STORED -> COPY_ -> COPY -> TYPE
TABLE -> LENLENS -> CODELENS -> LEN_
LEN_ -> LEN
Read deflate codes in fixed or dynamic block:
LEN -> LENEXT or LIT or TYPE
LENEXT -> DIST -> DISTEXT -> MATCH -> LEN
LIT -> LEN
Process trailer:
CHECK -> LENGTH -> DONE
*/
/* state maintained between inflate() calls. Approximately 10K bytes. */
struct inflate_state {
inflate_mode mode; /* current inflate mode */
int last; /* true if processing last block */
int wrap; /* bit 0 true for zlib, bit 1 true for gzip */
int havedict; /* true if dictionary provided */
int flags; /* gzip header method and flags (0 if zlib) */
unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */
unsigned long check; /* protected copy of check value */
unsigned long total; /* protected copy of output count */
gz_headerp head; /* where to save gzip header information */
/* sliding window */
unsigned wbits; /* log base 2 of requested window size */
unsigned wsize; /* window size or zero if not using window */
unsigned whave; /* valid bytes in the window */
unsigned wnext; /* window write index */
unsigned char FAR *window; /* allocated sliding window, if needed */
/* bit accumulator */
unsigned long hold; /* input bit accumulator */
unsigned bits; /* number of bits in "in" */
/* for string and stored block copying */
unsigned length; /* literal or length of data to copy */
unsigned offset; /* distance back to copy string from */
/* for table and code decoding */
unsigned extra; /* extra bits needed */
/* fixed and dynamic code tables */
code const FAR *lencode; /* starting table for length/literal codes */
code const FAR *distcode; /* starting table for distance codes */
unsigned lenbits; /* index bits for lencode */
unsigned distbits; /* index bits for distcode */
/* dynamic table building */
unsigned ncode; /* number of code length code lengths */
unsigned nlen; /* number of length code lengths */
unsigned ndist; /* number of distance code lengths */
unsigned have; /* number of code lengths in lens[] */
code FAR *next; /* next available space in codes[] */
unsigned short lens[320]; /* temporary storage for code lengths */
unsigned short work[288]; /* work area for code table building */
code codes[ENOUGH]; /* space for code tables */
int sane; /* if false, allow invalid distance too far */
int back; /* bits back of last unprocessed length/lit */
unsigned was; /* initial length of match */
};

306
fdbrpc/zlib/inftrees.c
View File

@ -1,306 +0,0 @@
/* inftrees.c -- generate Huffman trees for efficient decoding
* Copyright (C) 1995-2013 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
#include "zutil.h"
#include "inftrees.h"
#define MAXBITS 15
const char inflate_copyright[] =
" inflate 1.2.8 Copyright 1995-2013 Mark Adler ";
/*
If you use the zlib library in a product, an acknowledgment is welcome
in the documentation of your product. If for some reason you cannot
include such an acknowledgment, I would appreciate that you keep this
copyright string in the executable of your product.
*/
/*
Build a set of tables to decode the provided canonical Huffman code.
The code lengths are lens[0..codes-1]. The result starts at *table,
whose indices are 0..2^bits-1. work is a writable array of at least
lens shorts, which is used as a work area. type is the type of code
to be generated, CODES, LENS, or DISTS. On return, zero is success,
-1 is an invalid code, and +1 means that ENOUGH isn't enough. table
on return points to the next available entry's address. bits is the
requested root table index bits, and on return it is the actual root
table index bits. It will differ if the request is greater than the
longest code or if it is less than the shortest code.
*/
int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work)
codetype type;
unsigned short FAR *lens;
unsigned codes;
code FAR * FAR *table;
unsigned FAR *bits;
unsigned short FAR *work;
{
unsigned len; /* a code's length in bits */
unsigned sym; /* index of code symbols */
unsigned min, max; /* minimum and maximum code lengths */
unsigned root; /* number of index bits for root table */
unsigned curr; /* number of index bits for current table */
unsigned drop; /* code bits to drop for sub-table */
int left; /* number of prefix codes available */
unsigned used; /* code entries in table used */
unsigned huff; /* Huffman code */
unsigned incr; /* for incrementing code, index */
unsigned fill; /* index for replicating entries */
unsigned low; /* low bits for current root entry */
unsigned mask; /* mask for low root bits */
code here; /* table entry for duplication */
code FAR *next; /* next available space in table */
const unsigned short FAR *base; /* base value table to use */
const unsigned short FAR *extra; /* extra bits table to use */
int end; /* use base and extra for symbol > end */
unsigned short count[MAXBITS+1]; /* number of codes of each length */
unsigned short offs[MAXBITS+1]; /* offsets in table for each length */
static const unsigned short lbase[31] = { /* Length codes 257..285 base */
3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31,
35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0};
static const unsigned short lext[31] = { /* Length codes 257..285 extra */
16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18,
19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 72, 78};
static const unsigned short dbase[32] = { /* Distance codes 0..29 base */
1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145,
8193, 12289, 16385, 24577, 0, 0};
static const unsigned short dext[32] = { /* Distance codes 0..29 extra */
16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22,
23, 23, 24, 24, 25, 25, 26, 26, 27, 27,
28, 28, 29, 29, 64, 64};
/*
Process a set of code lengths to create a canonical Huffman code. The
code lengths are lens[0..codes-1]. Each length corresponds to the
symbols 0..codes-1. The Huffman code is generated by first sorting the
symbols by length from short to long, and retaining the symbol order
for codes with equal lengths. Then the code starts with all zero bits
for the first code of the shortest length, and the codes are integer
increments for the same length, and zeros are appended as the length
increases. For the deflate format, these bits are stored backwards
from their more natural integer increment ordering, and so when the
decoding tables are built in the large loop below, the integer codes
are incremented backwards.
This routine assumes, but does not check, that all of the entries in
lens[] are in the range 0..MAXBITS. The caller must assure this.
1..MAXBITS is interpreted as that code length. zero means that that
symbol does not occur in this code.
The codes are sorted by computing a count of codes for each length,
creating from that a table of starting indices for each length in the
sorted table, and then entering the symbols in order in the sorted
table. The sorted table is work[], with that space being provided by
the caller.
The length counts are used for other purposes as well, i.e. finding
the minimum and maximum length codes, determining if there are any
codes at all, checking for a valid set of lengths, and looking ahead
at length counts to determine sub-table sizes when building the
decoding tables.
*/
/* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */
for (len = 0; len <= MAXBITS; len++)
count[len] = 0;
for (sym = 0; sym < codes; sym++)
count[lens[sym]]++;
/* bound code lengths, force root to be within code lengths */
root = *bits;
for (max = MAXBITS; max >= 1; max--)
if (count[max] != 0) break;
if (root > max) root = max;
if (max == 0) { /* no symbols to code at all */
here.op = (unsigned char)64; /* invalid code marker */
here.bits = (unsigned char)1;
here.val = (unsigned short)0;
*(*table)++ = here; /* make a table to force an error */
*(*table)++ = here;
*bits = 1;
return 0; /* no symbols, but wait for decoding to report error */
}
for (min = 1; min < max; min++)
if (count[min] != 0) break;
if (root < min) root = min;
/* check for an over-subscribed or incomplete set of lengths */
left = 1;
for (len = 1; len <= MAXBITS; len++) {
left <<= 1;
left -= count[len];
if (left < 0) return -1; /* over-subscribed */
}
if (left > 0 && (type == CODES || max != 1))
return -1; /* incomplete set */
/* generate offsets into symbol table for each length for sorting */
offs[1] = 0;
for (len = 1; len < MAXBITS; len++)
offs[len + 1] = offs[len] + count[len];
/* sort symbols by length, by symbol order within each length */
for (sym = 0; sym < codes; sym++)
if (lens[sym] != 0) work[offs[lens[sym]]++] = (unsigned short)sym;
/*
Create and fill in decoding tables. In this loop, the table being
filled is at next and has curr index bits. The code being used is huff
with length len. That code is converted to an index by dropping drop
bits off of the bottom. For codes where len is less than drop + curr,
those top drop + curr - len bits are incremented through all values to
fill the table with replicated entries.
root is the number of index bits for the root table. When len exceeds
root, sub-tables are created pointed to by the root entry with an index
of the low root bits of huff. This is saved in low to check for when a
new sub-table should be started. drop is zero when the root table is
being filled, and drop is root when sub-tables are being filled.
When a new sub-table is needed, it is necessary to look ahead in the
code lengths to determine what size sub-table is needed. The length
counts are used for this, and so count[] is decremented as codes are
entered in the tables.
used keeps track of how many table entries have been allocated from the
provided *table space. It is checked for LENS and DIST tables against
the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in
the initial root table size constants. See the comments in inftrees.h
for more information.
sym increments through all symbols, and the loop terminates when
all codes of length max, i.e. all codes, have been processed. This
routine permits incomplete codes, so another loop after this one fills
in the rest of the decoding tables with invalid code markers.
*/
/* set up for code type */
switch (type) {
case CODES:
base = extra = work; /* dummy value--not used */
end = 19;
break;
case LENS:
base = lbase;
base -= 257;
extra = lext;
extra -= 257;
end = 256;
break;
default: /* DISTS */
base = dbase;
extra = dext;
end = -1;
}
/* initialize state for loop */
huff = 0; /* starting code */
sym = 0; /* starting code symbol */
len = min; /* starting code length */
next = *table; /* current table to fill in */
curr = root; /* current table index bits */
drop = 0; /* current bits to drop from code for index */
low = (unsigned)(-1); /* trigger new sub-table when len > root */
used = 1U << root; /* use root table entries */
mask = used - 1; /* mask for comparing low */
/* check available table space */
if ((type == LENS && used > ENOUGH_LENS) ||
(type == DISTS && used > ENOUGH_DISTS))
return 1;
/* process all codes and make table entries */
for (;;) {
/* create table entry */
here.bits = (unsigned char)(len - drop);
if ((int)(work[sym]) < end) {
here.op = (unsigned char)0;
here.val = work[sym];
}
else if ((int)(work[sym]) > end) {
here.op = (unsigned char)(extra[work[sym]]);
here.val = base[work[sym]];
}
else {
here.op = (unsigned char)(32 + 64); /* end of block */
here.val = 0;
}
/* replicate for those indices with low len bits equal to huff */
incr = 1U << (len - drop);
fill = 1U << curr;
min = fill; /* save offset to next table */
do {
fill -= incr;
next[(huff >> drop) + fill] = here;
} while (fill != 0);
/* backwards increment the len-bit code huff */
incr = 1U << (len - 1);
while (huff & incr)
incr >>= 1;
if (incr != 0) {
huff &= incr - 1;
huff += incr;
}
else
huff = 0;
/* go to next symbol, update count, len */
sym++;
if (--(count[len]) == 0) {
if (len == max) break;
len = lens[work[sym]];
}
/* create new sub-table if needed */
if (len > root && (huff & mask) != low) {
/* if first time, transition to sub-tables */
if (drop == 0)
drop = root;
/* increment past last table */
next += min; /* here min is 1 << curr */
/* determine length of next table */
curr = len - drop;
left = (int)(1 << curr);
while (curr + drop < max) {
left -= count[curr + drop];
if (left <= 0) break;
curr++;
left <<= 1;
}
/* check for enough space */
used += 1U << curr;
if ((type == LENS && used > ENOUGH_LENS) ||
(type == DISTS && used > ENOUGH_DISTS))
return 1;
/* point entry in root table to sub-table */
low = huff & mask;
(*table)[low].op = (unsigned char)curr;
(*table)[low].bits = (unsigned char)root;
(*table)[low].val = (unsigned short)(next - *table);
}
}
/* fill in remaining table entry if code is incomplete (guaranteed to have
at most one remaining entry, since if the code is incomplete, the
maximum code length that was allowed to get this far is one bit) */
if (huff != 0) {
here.op = (unsigned char)64; /* invalid code marker */
here.bits = (unsigned char)(len - drop);
here.val = (unsigned short)0;
next[huff] = here;
}
/* set return parameters */
*table += used;
*bits = root;
return 0;
}

62
fdbrpc/zlib/inftrees.h
View File

@ -1,62 +0,0 @@
/* inftrees.h -- header to use inftrees.c
* Copyright (C) 1995-2005, 2010 Mark Adler
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* WARNING: this file should *not* be used by applications. It is
part of the implementation of the compression library and is
subject to change. Applications should only use zlib.h.
*/
/* Structure for decoding tables. Each entry provides either the
information needed to do the operation requested by the code that
indexed that table entry, or it provides a pointer to another
table that indexes more bits of the code. op indicates whether
the entry is a pointer to another table, a literal, a length or
distance, an end-of-block, or an invalid code. For a table
pointer, the low four bits of op is the number of index bits of
that table. For a length or distance, the low four bits of op
is the number of extra bits to get after the code. bits is
the number of bits in this code or part of the code to drop off
of the bit buffer. val is the actual byte to output in the case
of a literal, the base length or distance, or the offset from
the current table to the next table. Each entry is four bytes. */
typedef struct {
unsigned char op; /* operation, extra bits, table bits */
unsigned char bits; /* bits in this part of the code */
unsigned short val; /* offset in table or code value */
} code;
/* op values as set by inflate_table():
00000000 - literal
0000tttt - table link, tttt != 0 is the number of table index bits
0001eeee - length or distance, eeee is the number of extra bits
01100000 - end of block
01000000 - invalid code
*/
/* Maximum size of the dynamic table. The maximum number of code structures is
1444, which is the sum of 852 for literal/length codes and 592 for distance
codes. These values were found by exhaustive searches using the program
examples/enough.c found in the zlib distribtution. The arguments to that
program are the number of symbols, the initial root table size, and the
maximum bit length of a code. "enough 286 9 15" for literal/length codes
returns returns 852, and "enough 30 6 15" for distance codes returns 592.
The initial root table size (9 or 6) is found in the fifth argument of the
inflate_table() calls in inflate.c and infback.c. If the root table size is
changed, then these maximum sizes would be need to be recalculated and
updated. */
#define ENOUGH_LENS 852
#define ENOUGH_DISTS 592
#define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS)
/* Type of code to build for inflate_table() */
typedef enum {
CODES,
LENS,
DISTS
} codetype;
int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens,
unsigned codes, code FAR * FAR *table,
unsigned FAR *bits, unsigned short FAR *work));

1226
fdbrpc/zlib/trees.c
View File

File diff suppressed because it is too large Load Diff

127
fdbrpc/zlib/trees.h
View File

@ -1,127 +0,0 @@
/* header created automatically with -DGEN_TREES_H */
local const ct_data static_ltree[L_CODES+2] = {
{{ 12},{ 8}}, {{140},{ 8}}, {{ 76},{ 8}}, {{204},{ 8}}, {{ 44},{ 8}},
{{172},{ 8}}, {{108},{ 8}}, {{236},{ 8}}, {{ 28},{ 8}}, {{156},{ 8}},
{{ 92},{ 8}}, {{220},{ 8}}, {{ 60},{ 8}}, {{188},{ 8}}, {{124},{ 8}},
{{252},{ 8}}, {{ 2},{ 8}}, {{130},{ 8}}, {{ 66},{ 8}}, {{194},{ 8}},
{{ 34},{ 8}}, {{162},{ 8}}, {{ 98},{ 8}}, {{226},{ 8}}, {{ 18},{ 8}},
{{146},{ 8}}, {{ 82},{ 8}}, {{210},{ 8}}, {{ 50},{ 8}}, {{178},{ 8}},
{{114},{ 8}}, {{242},{ 8}}, {{ 10},{ 8}}, {{138},{ 8}}, {{ 74},{ 8}},
{{202},{ 8}}, {{ 42},{ 8}}, {{170},{ 8}}, {{106},{ 8}}, {{234},{ 8}},
{{ 26},{ 8}}, {{154},{ 8}}, {{ 90},{ 8}}, {{218},{ 8}}, {{ 58},{ 8}},
{{186},{ 8}}, {{122},{ 8}}, {{250},{ 8}}, {{ 6},{ 8}}, {{134},{ 8}},
{{ 70},{ 8}}, {{198},{ 8}}, {{ 38},{ 8}}, {{166},{ 8}}, {{102},{ 8}},
{{230},{ 8}}, {{ 22},{ 8}}, {{150},{ 8}}, {{ 86},{ 8}}, {{214},{ 8}},
{{ 54},{ 8}}, {{182},{ 8}}, {{118},{ 8}}, {{246},{ 8}}, {{ 14},{ 8}},
{{142},{ 8}}, {{ 78},{ 8}}, {{206},{ 8}}, {{ 46},{ 8}}, {{174},{ 8}},
{{110},{ 8}}, {{238},{ 8}}, {{ 30},{ 8}}, {{158},{ 8}}, {{ 94},{ 8}},
{{222},{ 8}}, {{ 62},{ 8}}, {{190},{ 8}}, {{126},{ 8}}, {{254},{ 8}},
{{ 1},{ 8}}, {{129},{ 8}}, {{ 65},{ 8}}, {{193},{ 8}}, {{ 33},{ 8}},
{{161},{ 8}}, {{ 97},{ 8}}, {{225},{ 8}}, {{ 17},{ 8}}, {{145},{ 8}},
{{ 81},{ 8}}, {{209},{ 8}}, {{ 49},{ 8}}, {{177},{ 8}}, {{113},{ 8}},
{{241},{ 8}}, {{ 9},{ 8}}, {{137},{ 8}}, {{ 73},{ 8}}, {{201},{ 8}},
{{ 41},{ 8}}, {{169},{ 8}}, {{105},{ 8}}, {{233},{ 8}}, {{ 25},{ 8}},
{{153},{ 8}}, {{ 89},{ 8}}, {{217},{ 8}}, {{ 57},{ 8}}, {{185},{ 8}},
{{121},{ 8}}, {{249},{ 8}}, {{ 5},{ 8}}, {{133},{ 8}}, {{ 69},{ 8}},
{{197},{ 8}}, {{ 37},{ 8}}, {{165},{ 8}}, {{101},{ 8}}, {{229},{ 8}},
{{ 21},{ 8}}, {{149},{ 8}}, {{ 85},{ 8}}, {{213},{ 8}}, {{ 53},{ 8}},
{{181},{ 8}}, {{117},{ 8}}, {{245},{ 8}}, {{ 13},{ 8}}, {{141},{ 8}},
{{ 77},{ 8}}, {{205},{ 8}}, {{ 45},{ 8}}, {{173},{ 8}}, {{109},{ 8}},
{{237},{ 8}}, {{ 29},{ 8}}, {{157},{ 8}}, {{ 93},{ 8}}, {{221},{ 8}},
{{ 61},{ 8}}, {{189},{ 8}}, {{125},{ 8}}, {{253},{ 8}}, {{ 19},{ 9}},
{{275},{ 9}}, {{147},{ 9}}, {{403},{ 9}}, {{ 83},{ 9}}, {{339},{ 9}},
{{211},{ 9}}, {{467},{ 9}}, {{ 51},{ 9}}, {{307},{ 9}}, {{179},{ 9}},
{{435},{ 9}}, {{115},{ 9}}, {{371},{ 9}}, {{243},{ 9}}, {{499},{ 9}},
{{ 11},{ 9}}, {{267},{ 9}}, {{139},{ 9}}, {{395},{ 9}}, {{ 75},{ 9}},
{{331},{ 9}}, {{203},{ 9}}, {{459},{ 9}}, {{ 43},{ 9}}, {{299},{ 9}},
{{171},{ 9}}, {{427},{ 9}}, {{107},{ 9}}, {{363},{ 9}}, {{235},{ 9}},
{{491},{ 9}}, {{ 27},{ 9}}, {{283},{ 9}}, {{155},{ 9}}, {{411},{ 9}},
{{ 91},{ 9}}, {{347},{ 9}}, {{219},{ 9}}, {{475},{ 9}}, {{ 59},{ 9}},
{{315},{ 9}}, {{187},{ 9}}, {{443},{ 9}}, {{123},{ 9}}, {{379},{ 9}},
{{251},{ 9}}, {{507},{ 9}}, {{ 7},{ 9}}, {{263},{ 9}}, {{135},{ 9}},
{{391},{ 9}}, {{ 71},{ 9}}, {{327},{ 9}}, {{199},{ 9}}, {{455},{ 9}},
{{ 39},{ 9}}, {{295},{ 9}}, {{167},{ 9}}, {{423},{ 9}}, {{103},{ 9}},
{{359},{ 9}}, {{231},{ 9}}, {{487},{ 9}}, {{ 23},{ 9}}, {{279},{ 9}},
{{151},{ 9}}, {{407},{ 9}}, {{ 87},{ 9}}, {{343},{ 9}}, {{215},{ 9}},
{{471},{ 9}}, {{ 55},{ 9}}, {{311},{ 9}}, {{183},{ 9}}, {{439},{ 9}},
{{119},{ 9}}, {{375},{ 9}}, {{247},{ 9}}, {{503},{ 9}}, {{ 15},{ 9}},
{{271},{ 9}}, {{143},{ 9}}, {{399},{ 9}}, {{ 79},{ 9}}, {{335},{ 9}},
{{207},{ 9}}, {{463},{ 9}}, {{ 47},{ 9}}, {{303},{ 9}}, {{175},{ 9}},
{{431},{ 9}}, {{111},{ 9}}, {{367},{ 9}}, {{239},{ 9}}, {{495},{ 9}},
{{ 31},{ 9}}, {{287},{ 9}}, {{159},{ 9}}, {{415},{ 9}}, {{ 95},{ 9}},
{{351},{ 9}}, {{223},{ 9}}, {{479},{ 9}}, {{ 63},{ 9}}, {{319},{ 9}},
{{191},{ 9}}, {{447},{ 9}}, {{127},{ 9}}, {{383},{ 9}}, {{255},{ 9}},
{{511},{ 9}}, {{ 0},{ 7}}, {{ 64},{ 7}}, {{ 32},{ 7}}, {{ 96},{ 7}},
{{ 16},{ 7}}, {{ 80},{ 7}}, {{ 48},{ 7}}, {{112},{ 7}}, {{ 8},{ 7}},
{{ 72},{ 7}}, {{ 40},{ 7}}, {{104},{ 7}}, {{ 24},{ 7}}, {{ 88},{ 7}},
{{ 56},{ 7}}, {{120},{ 7}}, {{ 4},{ 7}}, {{ 68},{ 7}}, {{ 36},{ 7}},
{{100},{ 7}}, {{ 20},{ 7}}, {{ 84},{ 7}}, {{ 52},{ 7}}, {{116},{ 7}},
{{ 3},{ 8}}, {{131},{ 8}}, {{ 67},{ 8}}, {{195},{ 8}}, {{ 35},{ 8}},
{{163},{ 8}}, {{ 99},{ 8}}, {{227},{ 8}}
};
local const ct_data static_dtree[D_CODES] = {
{{ 0},{ 5}}, {{16},{ 5}}, {{ 8},{ 5}}, {{24},{ 5}}, {{ 4},{ 5}},
{{20},{ 5}}, {{12},{ 5}}, {{28},{ 5}}, {{ 2},{ 5}}, {{18},{ 5}},
{{10},{ 5}}, {{26},{ 5}}, {{ 6},{ 5}}, {{22},{ 5}}, {{14},{ 5}},
{{30},{ 5}}, {{ 1},{ 5}}, {{17},{ 5}}, {{ 9},{ 5}}, {{25},{ 5}},
{{ 5},{ 5}}, {{21},{ 5}}, {{13},{ 5}}, {{29},{ 5}}, {{ 3},{ 5}},
{{19},{ 5}}, {{11},{ 5}}, {{27},{ 5}}, {{ 7},{ 5}}, {{23},{ 5}}
};
const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = {
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8,
8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 16, 17,
18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29
};
const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= {
0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28
};
local const int base_length[LENGTH_CODES] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
64, 80, 96, 112, 128, 160, 192, 224, 0
};
local const int base_dist[D_CODES] = {
0, 1, 2, 3, 4, 6, 8, 12, 16, 24,
32, 48, 64, 96, 128, 192, 256, 384, 512, 768,
1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576
};

511
fdbrpc/zlib/zconf.h
View File

@ -1,511 +0,0 @@
/* zconf.h -- configuration of the zlib compression library
* Copyright (C) 1995-2013 Jean-loup Gailly.
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* @(#) $Id$ */
#ifndef ZCONF_H
#define ZCONF_H
/*
* If you *really* need a unique prefix for all types and library functions,
* compile with -DZ_PREFIX. The "standard" zlib should be compiled without it.
* Even better than compiling with -DZ_PREFIX would be to use configure to set
* this permanently in zconf.h using "./configure --zprefix".
*/
#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */
# define Z_PREFIX_SET
/* all linked symbols */
# define _dist_code z__dist_code
# define _length_code z__length_code
# define _tr_align z__tr_align
# define _tr_flush_bits z__tr_flush_bits
# define _tr_flush_block z__tr_flush_block
# define _tr_init z__tr_init
# define _tr_stored_block z__tr_stored_block
# define _tr_tally z__tr_tally
# define adler32 z_adler32
# define adler32_combine z_adler32_combine
# define adler32_combine64 z_adler32_combine64
# ifndef Z_SOLO
# define compress z_compress
# define compress2 z_compress2
# define compressBound z_compressBound
# endif
# define crc32 z_crc32
# define crc32_combine z_crc32_combine
# define crc32_combine64 z_crc32_combine64
# define deflate z_deflate
# define deflateBound z_deflateBound
# define deflateCopy z_deflateCopy
# define deflateEnd z_deflateEnd
# define deflateInit2_ z_deflateInit2_
# define deflateInit_ z_deflateInit_
# define deflateParams z_deflateParams
# define deflatePending z_deflatePending
# define deflatePrime z_deflatePrime
# define deflateReset z_deflateReset
# define deflateResetKeep z_deflateResetKeep
# define deflateSetDictionary z_deflateSetDictionary
# define deflateSetHeader z_deflateSetHeader
# define deflateTune z_deflateTune
# define deflate_copyright z_deflate_copyright
# define get_crc_table z_get_crc_table
# ifndef Z_SOLO
# define gz_error z_gz_error
# define gz_intmax z_gz_intmax
# define gz_strwinerror z_gz_strwinerror
# define gzbuffer z_gzbuffer
# define gzclearerr z_gzclearerr
# define gzclose z_gzclose
# define gzclose_r z_gzclose_r
# define gzclose_w z_gzclose_w
# define gzdirect z_gzdirect
# define gzdopen z_gzdopen
# define gzeof z_gzeof
# define gzerror z_gzerror
# define gzflush z_gzflush
# define gzgetc z_gzgetc
# define gzgetc_ z_gzgetc_
# define gzgets z_gzgets
# define gzoffset z_gzoffset
# define gzoffset64 z_gzoffset64
# define gzopen z_gzopen
# define gzopen64 z_gzopen64
# ifdef _WIN32
# define gzopen_w z_gzopen_w
# endif
# define gzprintf z_gzprintf
# define gzvprintf z_gzvprintf
# define gzputc z_gzputc
# define gzputs z_gzputs
# define gzread z_gzread
# define gzrewind z_gzrewind
# define gzseek z_gzseek
# define gzseek64 z_gzseek64
# define gzsetparams z_gzsetparams
# define gztell z_gztell
# define gztell64 z_gztell64
# define gzungetc z_gzungetc
# define gzwrite z_gzwrite
# endif
# define inflate z_inflate
# define inflateBack z_inflateBack
# define inflateBackEnd z_inflateBackEnd
# define inflateBackInit_ z_inflateBackInit_
# define inflateCopy z_inflateCopy
# define inflateEnd z_inflateEnd
# define inflateGetHeader z_inflateGetHeader
# define inflateInit2_ z_inflateInit2_
# define inflateInit_ z_inflateInit_
# define inflateMark z_inflateMark
# define inflatePrime z_inflatePrime
# define inflateReset z_inflateReset
# define inflateReset2 z_inflateReset2
# define inflateSetDictionary z_inflateSetDictionary
# define inflateGetDictionary z_inflateGetDictionary
# define inflateSync z_inflateSync
# define inflateSyncPoint z_inflateSyncPoint
# define inflateUndermine z_inflateUndermine
# define inflateResetKeep z_inflateResetKeep
# define inflate_copyright z_inflate_copyright
# define inflate_fast z_inflate_fast
# define inflate_table z_inflate_table
# ifndef Z_SOLO
# define uncompress z_uncompress
# endif
# define zError z_zError
# ifndef Z_SOLO
# define zcalloc z_zcalloc
# define zcfree z_zcfree
# endif
# define zlibCompileFlags z_zlibCompileFlags
# define zlibVersion z_zlibVersion
/* all zlib typedefs in zlib.h and zconf.h */
# define Byte z_Byte
# define Bytef z_Bytef
# define alloc_func z_alloc_func
# define charf z_charf
# define free_func z_free_func
# ifndef Z_SOLO
# define gzFile z_gzFile
# endif
# define gz_header z_gz_header
# define gz_headerp z_gz_headerp
# define in_func z_in_func
# define intf z_intf
# define out_func z_out_func
# define uInt z_uInt
# define uIntf z_uIntf
# define uLong z_uLong
# define uLongf z_uLongf
# define voidp z_voidp
# define voidpc z_voidpc
# define voidpf z_voidpf
/* all zlib structs in zlib.h and zconf.h */
# define gz_header_s z_gz_header_s
# define internal_state z_internal_state
#endif
#if defined(__MSDOS__) && !defined(MSDOS)
# define MSDOS
#endif
#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2)
# define OS2
#endif
#if defined(_WINDOWS) && !defined(WINDOWS)
# define WINDOWS
#endif
#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__)
# ifndef WIN32
# define WIN32
# endif
#endif
#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32)
# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__)
# ifndef SYS16BIT
# define SYS16BIT
# endif
# endif
#endif
/*
* Compile with -DMAXSEG_64K if the alloc function cannot allocate more
* than 64k bytes at a time (needed on systems with 16-bit int).
*/
#ifdef SYS16BIT
# define MAXSEG_64K
#endif
#ifdef MSDOS
# define UNALIGNED_OK
#endif
#ifdef __STDC_VERSION__
# ifndef STDC
# define STDC
# endif
# if __STDC_VERSION__ >= 199901L
# ifndef STDC99
# define STDC99
# endif
# endif
#endif
#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus))
# define STDC
#endif
#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__))
# define STDC
#endif
#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32))
# define STDC
#endif
#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__))
# define STDC
#endif
#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */
# define STDC
#endif
#ifndef STDC
# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */
# define const /* note: need a more gentle solution here */
# endif
#endif
#if defined(ZLIB_CONST) && !defined(z_const)
# define z_const const
#else
# define z_const
#endif
/* Some Mac compilers merge all .h files incorrectly: */
#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__)
# define NO_DUMMY_DECL
#endif
/* Maximum value for memLevel in deflateInit2 */
#ifndef MAX_MEM_LEVEL
# ifdef MAXSEG_64K
# define MAX_MEM_LEVEL 8
# else
# define MAX_MEM_LEVEL 9
# endif
#endif
/* Maximum value for windowBits in deflateInit2 and inflateInit2.
* WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
* created by gzip. (Files created by minigzip can still be extracted by
* gzip.)
*/
#ifndef MAX_WBITS
# define MAX_WBITS 15 /* 32K LZ77 window */
#endif
/* The memory requirements for deflate are (in bytes):
(1 << (windowBits+2)) + (1 << (memLevel+9))
that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values)
plus a few kilobytes for small objects. For example, if you want to reduce
the default memory requirements from 256K to 128K, compile with
make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
Of course this will generally degrade compression (there's no free lunch).
The memory requirements for inflate are (in bytes) 1 << windowBits
that is, 32K for windowBits=15 (default value) plus a few kilobytes
for small objects.
*/
/* Type declarations */
#ifndef OF /* function prototypes */
# ifdef STDC
# define OF(args) args
# else
# define OF(args) ()
# endif
#endif
#ifndef Z_ARG /* function prototypes for stdarg */
# if defined(STDC) || defined(Z_HAVE_STDARG_H)
# define Z_ARG(args) args
# else
# define Z_ARG(args) ()
# endif
#endif
/* The following definitions for FAR are needed only for MSDOS mixed
* model programming (small or medium model with some far allocations).
* This was tested only with MSC; for other MSDOS compilers you may have
* to define NO_MEMCPY in zutil.h. If you don't need the mixed model,
* just define FAR to be empty.
*/
#ifdef SYS16BIT
# if defined(M_I86SM) || defined(M_I86MM)
/* MSC small or medium model */
# define SMALL_MEDIUM
# ifdef _MSC_VER
# define FAR _far
# else
# define FAR far
# endif
# endif
# if (defined(__SMALL__) || defined(__MEDIUM__))
/* Turbo C small or medium model */
# define SMALL_MEDIUM
# ifdef __BORLANDC__
# define FAR _far
# else
# define FAR far
# endif
# endif
#endif
#if defined(WINDOWS) || defined(WIN32)
/* If building or using zlib as a DLL, define ZLIB_DLL.
* This is not mandatory, but it offers a little performance increase.
*/
# ifdef ZLIB_DLL
# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500))
# ifdef ZLIB_INTERNAL
# define ZEXTERN extern __declspec(dllexport)
# else
# define ZEXTERN extern __declspec(dllimport)
# endif
# endif
# endif /* ZLIB_DLL */
/* If building or using zlib with the WINAPI/WINAPIV calling convention,
* define ZLIB_WINAPI.
* Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
*/
# ifdef ZLIB_WINAPI
# ifdef FAR
# undef FAR
# endif
# include <windows.h>
/* No need for _export, use ZLIB.DEF instead. */
/* For complete Windows compatibility, use WINAPI, not __stdcall. */
# define ZEXPORT WINAPI
# ifdef WIN32
# define ZEXPORTVA WINAPIV
# else
# define ZEXPORTVA FAR CDECL
# endif
# endif
#endif
#if defined (__BEOS__)
# ifdef ZLIB_DLL
# ifdef ZLIB_INTERNAL
# define ZEXPORT __declspec(dllexport)
# define ZEXPORTVA __declspec(dllexport)
# else
# define ZEXPORT __declspec(dllimport)
# define ZEXPORTVA __declspec(dllimport)
# endif
# endif
#endif
#ifndef ZEXTERN
# define ZEXTERN extern
#endif
#ifndef ZEXPORT
# define ZEXPORT
#endif
#ifndef ZEXPORTVA
# define ZEXPORTVA
#endif
#ifndef FAR
# define FAR
#endif
#if !defined(__MACTYPES__)
typedef unsigned char Byte; /* 8 bits */
#endif
typedef unsigned int uInt; /* 16 bits or more */
typedef unsigned long uLong; /* 32 bits or more */
#ifdef SMALL_MEDIUM
/* Borland C/C++ and some old MSC versions ignore FAR inside typedef */
# define Bytef Byte FAR
#else
typedef Byte FAR Bytef;
#endif
typedef char FAR charf;
typedef int FAR intf;
typedef uInt FAR uIntf;
typedef uLong FAR uLongf;
#ifdef STDC
typedef void const *voidpc;
typedef void FAR *voidpf;
typedef void *voidp;
#else
typedef Byte const *voidpc;
typedef Byte FAR *voidpf;
typedef Byte *voidp;
#endif
#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC)
# include <limits.h>
# if (UINT_MAX == 0xffffffffUL)
# define Z_U4 unsigned
# elif (ULONG_MAX == 0xffffffffUL)
# define Z_U4 unsigned long
# elif (USHRT_MAX == 0xffffffffUL)
# define Z_U4 unsigned short
# endif
#endif
#ifdef Z_U4
typedef Z_U4 z_crc_t;
#else
typedef unsigned long z_crc_t;
#endif
#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */
# define Z_HAVE_UNISTD_H
#endif
#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */
# define Z_HAVE_STDARG_H
#endif
#ifdef STDC
# ifndef Z_SOLO
# include <sys/types.h> /* for off_t */
# endif
#endif
#if defined(STDC) || defined(Z_HAVE_STDARG_H)
# ifndef Z_SOLO
# include <stdarg.h> /* for va_list */
# endif
#endif
#ifdef _WIN32
# ifndef Z_SOLO
# include <stddef.h> /* for wchar_t */
# endif
#endif
/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and
* "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even
* though the former does not conform to the LFS document), but considering
* both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as
* equivalently requesting no 64-bit operations
*/
#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1
# undef _LARGEFILE64_SOURCE
#endif
#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H)
# define Z_HAVE_UNISTD_H
#endif
#ifndef Z_SOLO
# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
# include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
# ifdef VMS
# include <unixio.h> /* for off_t */
# endif
# ifndef z_off_t
# define z_off_t off_t
# endif
# endif
#endif
#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0
# define Z_LFS64
#endif
#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64)
# define Z_LARGE64
#endif
#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64)
# define Z_WANT64
#endif
#if !defined(SEEK_SET) && !defined(Z_SOLO)
# define SEEK_SET 0 /* Seek from beginning of file. */
# define SEEK_CUR 1 /* Seek from current position. */
# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */
#endif
#ifndef z_off_t
# define z_off_t long
#endif
#if !defined(_WIN32) && defined(Z_LARGE64)
# define z_off64_t off64_t
#else
# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO)
# define z_off64_t __int64
# else
# define z_off64_t z_off_t
# endif
#endif
/* MVS linker does not support external names larger than 8 bytes */
#if defined(__MVS__)
#pragma map(deflateInit_,"DEIN")
#pragma map(deflateInit2_,"DEIN2")
#pragma map(deflateEnd,"DEEND")
#pragma map(deflateBound,"DEBND")
#pragma map(inflateInit_,"ININ")
#pragma map(inflateInit2_,"ININ2")
#pragma map(inflateEnd,"INEND")
#pragma map(inflateSync,"INSY")
#pragma map(inflateSetDictionary,"INSEDI")
#pragma map(compressBound,"CMBND")
#pragma map(inflate_table,"INTABL")
#pragma map(inflate_fast,"INFA")
#pragma map(inflate_copyright,"INCOPY")
#endif
#endif /* ZCONF_H */

1768
fdbrpc/zlib/zlib.h
View File

File diff suppressed because it is too large Load Diff

323
fdbrpc/zlib/zutil.c
View File

@ -1,323 +0,0 @@
/* zutil.c -- target dependent utility functions for the compression library
* Copyright (C) 1995-2005, 2010, 2011, 2012 Jean-loup Gailly.
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* @(#) $Id$ */
#include "zutil.h"
#ifndef Z_SOLO
# include "gzguts.h"
#endif
#ifndef NO_DUMMY_DECL
struct internal_state {int dummy;}; /* for buggy compilers */
#endif
z_const char * const z_errmsg[10] = {
"need dictionary", /* Z_NEED_DICT 2 */
"stream end", /* Z_STREAM_END 1 */
"", /* Z_OK 0 */
"file error", /* Z_ERRNO (-1) */
"stream error", /* Z_STREAM_ERROR (-2) */
"data error", /* Z_DATA_ERROR (-3) */
"insufficient memory", /* Z_MEM_ERROR (-4) */
"buffer error", /* Z_BUF_ERROR (-5) */
"incompatible version",/* Z_VERSION_ERROR (-6) */
""};
const char * ZEXPORT zlibVersion()
{
return ZLIB_VERSION;
}
uLong ZEXPORT zlibCompileFlags()
{
uLong flags;
flags = 0;
switch ((int)(sizeof(uInt))) {
case 2: break;
case 4: flags += 1; break;
case 8: flags += 2; break;
default: flags += 3;
}
switch ((int)(sizeof(uLong))) {
case 2: break;
case 4: flags += 1 << 2; break;
case 8: flags += 2 << 2; break;
default: flags += 3 << 2;
}
switch ((int)(sizeof(voidpf))) {
case 2: break;
case 4: flags += 1 << 4; break;
case 8: flags += 2 << 4; break;
default: flags += 3 << 4;
}
switch ((int)(sizeof(z_off_t))) {
case 2: break;
case 4: flags += 1 << 6; break;
case 8: flags += 2 << 6; break;
default: flags += 3 << 6;
}
#ifdef DEBUG
flags += 1 << 8;
#endif
#if defined(ASMV) || defined(ASMINF)
flags += 1 << 9;
#endif
#ifdef ZLIB_WINAPI
flags += 1 << 10;
#endif
#ifdef BUILDFIXED
flags += 1 << 12;
#endif
#ifdef DYNAMIC_CRC_TABLE
flags += 1 << 13;
#endif
#ifdef NO_GZCOMPRESS
flags += 1L << 16;
#endif
#ifdef NO_GZIP
flags += 1L << 17;
#endif
#ifdef PKZIP_BUG_WORKAROUND
flags += 1L << 20;
#endif
#ifdef FASTEST
flags += 1L << 21;
#endif
#if defined(STDC) || defined(Z_HAVE_STDARG_H)
# ifdef NO_vsnprintf
flags += 1L << 25;
# ifdef HAS_vsprintf_void
flags += 1L << 26;
# endif
# else
# ifdef HAS_vsnprintf_void
flags += 1L << 26;
# endif
# endif
#else
flags += 1L << 24;
# ifdef NO_snprintf
flags += 1L << 25;
# ifdef HAS_sprintf_void
flags += 1L << 26;
# endif
# else
# ifdef HAS_snprintf_void
flags += 1L << 26;
# endif
# endif
#endif
return flags;
}
#ifdef DEBUG
# ifndef verbose
# define verbose 0
# endif
int ZLIB_INTERNAL z_verbose = verbose;
void ZLIB_INTERNAL z_error (m)
char *m;
{
criticalError(FDB_EXIT_ABORT, "zlibError", m);
}
#endif
/* exported to allow conversion of error code to string for compress() and
* uncompress()
*/
const char * ZEXPORT zError(err)
int err;
{
return ERR_MSG(err);
}
#if defined(_WIN32_WCE)
/* The Microsoft C Run-Time Library for Windows CE doesn't have
* errno. We define it as a global variable to simplify porting.
* Its value is always 0 and should not be used.
*/
int errno = 0;
#endif
#ifndef HAVE_MEMCPY
void ZLIB_INTERNAL zmemcpy(dest, source, len)
Bytef* dest;
const Bytef* source;
uInt len;
{
if (len == 0) return;
do {
*dest++ = *source++; /* ??? to be unrolled */
} while (--len != 0);
}
int ZLIB_INTERNAL zmemcmp(s1, s2, len)
const Bytef* s1;
const Bytef* s2;
uInt len;
{
uInt j;
for (j = 0; j < len; j++) {
if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1;
}
return 0;
}
void ZLIB_INTERNAL zmemzero(dest, len)
Bytef* dest;
uInt len;
{
if (len == 0) return;
do {
*dest++ = 0; /* ??? to be unrolled */
} while (--len != 0);
}
#endif
#ifndef Z_SOLO
#ifdef SYS16BIT
#ifdef __TURBOC__
/* Turbo C in 16-bit mode */
# define MY_ZCALLOC
/* Turbo C malloc() does not allow dynamic allocation of 64K bytes
* and farmalloc(64K) returns a pointer with an offset of 8, so we
* must fix the pointer. Warning: the pointer must be put back to its
* original form in order to free it, use zcfree().
*/
#define MAX_PTR 10
/* 10*64K = 640K */
local int next_ptr = 0;
typedef struct ptr_table_s {
voidpf org_ptr;
voidpf new_ptr;
} ptr_table;
local ptr_table table[MAX_PTR];
/* This table is used to remember the original form of pointers
* to large buffers (64K). Such pointers are normalized with a zero offset.
* Since MSDOS is not a preemptive multitasking OS, this table is not
* protected from concurrent access. This hack doesn't work anyway on
* a protected system like OS/2. Use Microsoft C instead.
*/
voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size)
{
voidpf buf = opaque; /* just to make some compilers happy */
ulg bsize = (ulg)items*size;
/* If we allocate less than 65520 bytes, we assume that farmalloc
* will return a usable pointer which doesn't have to be normalized.
*/
if (bsize < 65520L) {
buf = farmalloc(bsize);
if (*(ush*)&buf != 0) return buf;
} else {
buf = farmalloc(bsize + 16L);
}
if (buf == NULL || next_ptr >= MAX_PTR) return NULL;
table[next_ptr].org_ptr = buf;
/* Normalize the pointer to seg:0 */
*((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4;
*(ush*)&buf = 0;
table[next_ptr++].new_ptr = buf;
return buf;
}
void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
{
int n;
if (*(ush*)&ptr != 0) { /* object < 64K */
farfree(ptr);
return;
}
/* Find the original pointer */
for (n = 0; n < next_ptr; n++) {
if (ptr != table[n].new_ptr) continue;
farfree(table[n].org_ptr);
while (++n < next_ptr) {
table[n-1] = table[n];
}
next_ptr--;
return;
}
ptr = opaque; /* just to make some compilers happy */
Assert(0, "zcfree: ptr not found");
}
#endif /* __TURBOC__ */
#ifdef M_I86
/* Microsoft C in 16-bit mode */
# define MY_ZCALLOC
#if (!defined(_MSC_VER) || (_MSC_VER <= 600))
# define _halloc halloc
# define _hfree hfree
#endif
voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size)
{
if (opaque) opaque = 0; /* to make compiler happy */
return _halloc((long)items, size);
}
void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
{
if (opaque) opaque = 0; /* to make compiler happy */
_hfree(ptr);
}
#endif /* M_I86 */
#endif /* SYS16BIT */
#ifndef MY_ZCALLOC /* Any system without a special alloc function */
#ifndef STDC
extern voidp malloc OF((uInt size));
extern voidp calloc OF((uInt items, uInt size));
extern void free OF((voidpf ptr));
#endif
voidpf ZLIB_INTERNAL zcalloc (opaque, items, size)
voidpf opaque;
unsigned items;
unsigned size;
{
if (opaque) items += size - size; /* make compiler happy */
return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) :
(voidpf)calloc(items, size);
}
void ZLIB_INTERNAL zcfree (opaque, ptr)
voidpf opaque;
voidpf ptr;
{
free(ptr);
if (opaque) return; /* make compiler happy */
}
#endif /* MY_ZCALLOC */
#endif /* !Z_SOLO */

253
fdbrpc/zlib/zutil.h
View File

@ -1,253 +0,0 @@
/* zutil.h -- internal interface and configuration of the compression library
* Copyright (C) 1995-2013 Jean-loup Gailly.
* For conditions of distribution and use, see copyright notice in zlib.h
*/
/* WARNING: this file should *not* be used by applications. It is
part of the implementation of the compression library and is
subject to change. Applications should only use zlib.h.
*/
/* @(#) $Id$ */
#ifndef ZUTIL_H
#define ZUTIL_H
#ifdef HAVE_HIDDEN
# define ZLIB_INTERNAL __attribute__((visibility ("hidden")))
#else
# define ZLIB_INTERNAL
#endif
#include "zlib.h"
#if defined(STDC) && !defined(Z_SOLO)
# if !(defined(_WIN32_WCE) && defined(_MSC_VER))
# include <stddef.h>
# endif
# include <string.h>
# include <stdlib.h>
#endif
#ifdef Z_SOLO
typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */
#endif
#ifndef local
# define local static
#endif
/* compile with -Dlocal if your debugger can't find static symbols */
typedef unsigned char uch;
typedef uch FAR uchf;
typedef unsigned short ush;
typedef ush FAR ushf;
typedef unsigned long ulg;
extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
/* (size given to avoid silly warnings with Visual C++) */
#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)]
#define ERR_RETURN(strm,err) \
return (strm->msg = ERR_MSG(err), (err))
/* To be used only when the state is known to be valid */
/* common constants */
#ifndef DEF_WBITS
# define DEF_WBITS MAX_WBITS
#endif
/* default windowBits for decompression. MAX_WBITS is for compression only */
#if MAX_MEM_LEVEL >= 8
# define DEF_MEM_LEVEL 8
#else
# define DEF_MEM_LEVEL MAX_MEM_LEVEL
#endif
/* default memLevel */
#define STORED_BLOCK 0
#define STATIC_TREES 1
#define DYN_TREES 2
/* The three kinds of block type */
#define MIN_MATCH 3
#define MAX_MATCH 258
/* The minimum and maximum match lengths */
#define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */
/* target dependencies */
#if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32))
# define OS_CODE 0x00
# ifndef Z_SOLO
# if defined(__TURBOC__) || defined(__BORLANDC__)
# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__))
/* Allow compilation with ANSI keywords only enabled */
void _Cdecl farfree( void *block );
void *_Cdecl farmalloc( unsigned long nbytes );
# else
# include <alloc.h>
# endif
# else /* MSC or DJGPP */
# include <malloc.h>
# endif
# endif
#endif
#ifdef AMIGA
# define OS_CODE 0x01
#endif
#if defined(VAXC) || defined(VMS)
# define OS_CODE 0x02
# define F_OPEN(name, mode) \
fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512")
#endif
#if defined(ATARI) || defined(atarist)
# define OS_CODE 0x05
#endif
#ifdef OS2
# define OS_CODE 0x06
# if defined(M_I86) && !defined(Z_SOLO)
# include <malloc.h>
# endif
#endif
#if defined(MACOS) || defined(TARGET_OS_MAC)
# define OS_CODE 0x07
# ifndef Z_SOLO
# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os
# include <unix.h> /* for fdopen */
# else
# ifndef fdopen
# define fdopen(fd,mode) NULL /* No fdopen() */
# endif
# endif
# endif
#endif
#ifdef TOPS20
# define OS_CODE 0x0a
#endif
#ifdef WIN32
# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */
# define OS_CODE 0x0b
# endif
#endif
#ifdef __50SERIES /* Prime/PRIMOS */
# define OS_CODE 0x0f
#endif
#if defined(_BEOS_) || defined(RISCOS)
# define fdopen(fd,mode) NULL /* No fdopen() */
#endif
#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX
# if defined(_WIN32_WCE)
# define fdopen(fd,mode) NULL /* No fdopen() */
# ifndef _PTRDIFF_T_DEFINED
typedef int ptrdiff_t;
# define _PTRDIFF_T_DEFINED
# endif
# else
# define fdopen(fd,type) _fdopen(fd,type)
# endif
#endif
#if defined(__BORLANDC__) && !defined(MSDOS)
#pragma warn -8004
#pragma warn -8008
#pragma warn -8066
#endif
/* provide prototypes for these when building zlib without LFS */
#if !defined(_WIN32) && \
(!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0)
ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
#endif
/* common defaults */
#ifndef OS_CODE
# define OS_CODE 0x03 /* assume Unix */
#endif
#ifndef F_OPEN
# define F_OPEN(name, mode) fopen((name), (mode))
#endif
/* functions */
#if defined(pyr) || defined(Z_SOLO)
# define NO_MEMCPY
#endif
#if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__)
/* Use our own functions for small and medium model with MSC <= 5.0.
* You may have to use the same strategy for Borland C (untested).
* The __SC__ check is for Symantec.
*/
# define NO_MEMCPY
#endif
#if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY)
# define HAVE_MEMCPY
#endif
#ifdef HAVE_MEMCPY
# ifdef SMALL_MEDIUM /* MSDOS small or medium model */
# define zmemcpy _fmemcpy
# define zmemcmp _fmemcmp
# define zmemzero(dest, len) _fmemset(dest, 0, len)
# else
# define zmemcpy memcpy
# define zmemcmp memcmp
# define zmemzero(dest, len) memset(dest, 0, len)
# endif
#else
void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len));
int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len));
void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len));
#endif
/* Diagnostic functions */
#ifdef DEBUG
# include <stdio.h>
extern int ZLIB_INTERNAL z_verbose;
extern void ZLIB_INTERNAL z_error OF((char *m));
# define Assert(cond,msg) {if(!(cond)) z_error(msg);}
# define Trace(x) {if (z_verbose>=0) fprintf x ;}
# define Tracev(x) {if (z_verbose>0) fprintf x ;}
# define Tracevv(x) {if (z_verbose>1) fprintf x ;}
# define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;}
# define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;}
#else
# define Assert(cond,msg)
# define Trace(x)
# define Tracev(x)
# define Tracevv(x)
# define Tracec(c,x)
# define Tracecv(c,x)
#endif
#ifndef Z_SOLO
voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items,
unsigned size));
void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr));
#endif
#define ZALLOC(strm, items, size) \
(*((strm)->zalloc))((strm)->opaque, (items), (size))
#define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr))
#define TRY_FREE(s, p) {if (p) ZFREE(s, p);}
/* Reverse the bytes in a 32-bit value */
#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \
(((q) & 0xff00) << 8) + (((q) & 0xff) << 24))
#endif /* ZUTIL_H */

54
fdbserver/ApplyMetadataMutation.cpp
View File

@ -45,9 +45,14 @@ Reference<StorageInfo> getStorageInfo(UID id, std::map<UID, Reference<StorageInf
// It is incredibly important that any modifications to txnStateStore are done in such a way that
// the same operations will be done on all proxies at the same time. Otherwise, the data stored in
// txnStateStore will become corrupted.
void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRef> const& mutations, IKeyValueStore* txnStateStore, LogPushData* toCommit, bool *confChange, Reference<ILogSystem> logSystem, Version popVersion,
KeyRangeMap<std::set<Key> >* vecBackupKeys, KeyRangeMap<ServerCacheInfo>* keyInfo, KeyRangeMap<bool>* cacheInfo, std::map<Key, applyMutationsData>* uid_applyMutationsData, RequestStream<CommitTransactionRequest> commit,
Database cx, NotifiedVersion* commitVersion, std::map<UID, Reference<StorageInfo>>* storageCache, std::map<Tag, Version>* tag_popped, bool initialCommit ) {
void applyMetadataMutations(UID const& dbgid, Arena& arena, VectorRef<MutationRef> const& mutations,
IKeyValueStore* txnStateStore, LogPushData* toCommit, bool& confChange,
Reference<ILogSystem> logSystem, Version popVersion,
KeyRangeMap<std::set<Key>>* vecBackupKeys, KeyRangeMap<ServerCacheInfo>* keyInfo,
KeyRangeMap<bool>* cacheInfo, std::map<Key, ApplyMutationsData>* uid_applyMutationsData,
RequestStream<CommitTransactionRequest> commit, Database cx, NotifiedVersion* commitVersion,
std::map<UID, Reference<StorageInfo>>* storageCache, std::map<Tag, Version>* tag_popped,
bool initialCommit) {
//std::map<keyRef, vector<uint16_t>> cacheRangeInfo;
std::map<KeyRef, MutationRef> cachedRangeInfo;
for (auto const& m : mutations) {
@ -175,7 +180,7 @@ void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRe
.detail("M", m.toString())
.detail("PrevValue", t.present() ? t.get() : LiteralStringRef("(none)"))
.detail("ToCommit", toCommit!=nullptr);
if(confChange) *confChange = true;
confChange = true;
}
}
if(!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2));
@ -293,7 +298,7 @@ void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRe
Version requested = BinaryReader::fromStringRef<Version>(m.param2, Unversioned());
TraceEvent("MinRequiredCommitVersion", dbgid).detail("Min", requested).detail("Current", popVersion).detail("HasConf", !!confChange);
if(!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2));
if (confChange) *confChange = true;
confChange = true;
TEST(true); // Recovering at a higher version.
}
}
@ -313,7 +318,7 @@ void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRe
if(!initialCommit) txnStateStore->clear(range & configKeys);
if(!excludedServersKeys.contains(range) && !failedServersKeys.contains(range)) {
TraceEvent("MutationRequiresRestart", dbgid).detail("M", m.toString());
if(confChange) *confChange = true;
confChange = true;
}
}
if ( serverListKeys.intersects( range )) {
@ -329,11 +334,14 @@ void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRe
auto serverKeysCleared = txnStateStore->readRange( range & serverTagKeys ).get(); // read is expected to be immediately available
for(auto &kv : serverKeysCleared) {
Tag tag = decodeServerTagValue(kv.value);
TraceEvent("ServerTagRemove").detail("PopVersion", popVersion).detail("Tag", tag.toString()).detail("Server", decodeServerTagKey(kv.key));
logSystem->pop( popVersion, decodeServerTagValue(kv.value) );
TraceEvent("ServerTagRemove")
.detail("PopVersion", popVersion)
.detail("Tag", tag.toString())
.detail("Server", decodeServerTagKey(kv.key));
logSystem->pop(popVersion, decodeServerTagValue(kv.value));
(*tag_popped)[tag] = popVersion;
if(toCommit) {
if (toCommit) {
MutationRef privatized = m;
privatized.param1 = kv.key.withPrefix(systemKeys.begin, arena);
privatized.param2 = keyAfter(kv.key, arena).withPrefix(systemKeys.begin, arena);
@ -536,3 +544,31 @@ void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<MutationRe
}
}
}
void applyMetadataMutations(ProxyCommitData& proxyCommitData, Arena& arena, Reference<ILogSystem> logSystem,
const VectorRef<MutationRef>& mutations, LogPushData* toCommit, bool& confChange,
Version popVersion, bool initialCommit) {
std::map<Key, ApplyMutationsData>* uid_applyMutationsData = nullptr;
if (proxyCommitData.firstProxy) {
uid_applyMutationsData = &proxyCommitData.uid_applyMutationsData;
}
applyMetadataMutations(proxyCommitData.dbgid, arena, mutations, proxyCommitData.txnStateStore, toCommit, confChange,
logSystem, popVersion, &proxyCommitData.vecBackupKeys, &proxyCommitData.keyInfo,
&proxyCommitData.cacheInfo, uid_applyMutationsData, proxyCommitData.commit,
proxyCommitData.cx, &proxyCommitData.committedVersion, &proxyCommitData.storageCache,
&proxyCommitData.tag_popped, initialCommit);
}
void applyMetadataMutations(const UID& dbgid, Arena& arena, const VectorRef<MutationRef>& mutations,
IKeyValueStore* txnStateStore) {
bool confChange; // Dummy variable, not used.
applyMetadataMutations(dbgid, arena, mutations, txnStateStore, /* toCommit= */ nullptr, confChange,
Reference<ILogSystem>(), /* popVersion= */ 0, /* vecBackupKeys= */ nullptr,
/* keyInfo= */ nullptr, /* cacheInfo= */ nullptr, /* uid_applyMutationsData= */ nullptr,
RequestStream<CommitTransactionRequest>(), Database(), /* commitVersion= */ nullptr,
/* storageCache= */ nullptr, /* tag_popped= */ nullptr, /* initialCommit= */ false);
}

Some files were not shown because too many files have changed in this diff Show More